From ca9112a424ff68ec4f2ef67b47122f7d61412964 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 15 Aug 2016 13:49:49 -0400 Subject: [PATCH 001/871] Stamp HEAD as 10devel. This is a good bit more complicated than the average new-version stamping commit, because it includes various adjustments in pursuit of changing from three-part to two-part version numbers. It's likely some further work will be needed around that change; but this is enough to get through the regression tests, at least in Unix builds. Peter Eisentraut and Tom Lane --- configure | 20 ++++++++++---------- configure.in | 4 ++-- doc/bug.template | 2 +- doc/src/sgml/runtime.sgml | 31 ++++++++++++++++++++----------- src/backend/catalog/genbki.pl | 4 ++-- src/backend/utils/init/miscinit.c | 26 +++++++++++++------------- src/bin/pg_upgrade/check.c | 6 +++--- src/bin/pg_upgrade/server.c | 2 +- src/include/pg_config.h.win32 | 8 ++++---- src/interfaces/libpq/libpq.rc.in | 8 ++++---- src/port/win32ver.rc | 4 ++-- src/tools/msvc/Solution.pm | 6 +++--- src/tools/version_stamp.pl | 14 ++++++-------- 13 files changed, 71 insertions(+), 64 deletions(-) diff --git a/configure b/configure index 7244c755a7..45c8eefad7 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for PostgreSQL 9.6beta4. +# Generated by GNU Autoconf 2.69 for PostgreSQL 10devel. # # Report bugs to . # @@ -582,8 +582,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='PostgreSQL' PACKAGE_TARNAME='postgresql' -PACKAGE_VERSION='9.6beta4' -PACKAGE_STRING='PostgreSQL 9.6beta4' +PACKAGE_VERSION='10devel' +PACKAGE_STRING='PostgreSQL 10devel' PACKAGE_BUGREPORT='pgsql-bugs@postgresql.org' PACKAGE_URL='' @@ -1398,7 +1398,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures PostgreSQL 9.6beta4 to adapt to many kinds of systems. +\`configure' configures PostgreSQL 10devel to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1463,7 +1463,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of PostgreSQL 9.6beta4:";; + short | recursive ) echo "Configuration of PostgreSQL 10devel:";; esac cat <<\_ACEOF @@ -1615,7 +1615,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -PostgreSQL configure 9.6beta4 +PostgreSQL configure 10devel generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2326,7 +2326,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by PostgreSQL $as_me 9.6beta4, which was +It was created by PostgreSQL $as_me 10devel, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2711,7 +2711,7 @@ ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. configure_args=$ac_configure_args -PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\.[0-9][0-9]*\)'` +PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'` cat >>confdefs.h <<_ACEOF @@ -16433,7 +16433,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by PostgreSQL $as_me 9.6beta4, which was +This file was extended by PostgreSQL $as_me 10devel, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -16503,7 +16503,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -PostgreSQL config.status 9.6beta4 +PostgreSQL config.status 10devel configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.in b/configure.in index 598fbd8f64..c878b4e377 100644 --- a/configure.in +++ b/configure.in @@ -17,7 +17,7 @@ dnl Read the Autoconf manual for details. dnl m4_pattern_forbid(^PGAC_)dnl to catch undefined macros -AC_INIT([PostgreSQL], [9.6beta4], [pgsql-bugs@postgresql.org]) +AC_INIT([PostgreSQL], [10devel], [pgsql-bugs@postgresql.org]) m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. Untested combinations of 'autoconf' and PostgreSQL versions are not @@ -29,7 +29,7 @@ AC_CONFIG_AUX_DIR(config) AC_PREFIX_DEFAULT(/usr/local/pgsql) AC_SUBST(configure_args, [$ac_configure_args]) -[PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\.[0-9][0-9]*\)'`] +[PG_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\)'`] AC_SUBST(PG_MAJORVERSION) AC_DEFINE_UNQUOTED(PG_MAJORVERSION, "$PG_MAJORVERSION", [PostgreSQL major version as a string]) diff --git a/doc/bug.template b/doc/bug.template index 5534772196..8e7401e1ce 100644 --- a/doc/bug.template +++ b/doc/bug.template @@ -27,7 +27,7 @@ System Configuration: Operating System (example: Linux 2.4.18) : - PostgreSQL version (example: PostgreSQL 9.6beta4): PostgreSQL 9.6beta4 + PostgreSQL version (example: PostgreSQL 10devel): PostgreSQL 10devel Compiler used (example: gcc 3.3.5) : diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 8ba95e1b84..66fbe441ac 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1601,17 +1601,26 @@ $ kill -INT `head -1 /usr/local/pgsql/data/postmaster.pid` - PostgreSQL major versions are represented by the - first two digit groups of the version number, e.g., 8.4. - PostgreSQL minor versions are represented by the - third group of version digits, e.g., 8.4.2 is the second minor - release of 8.4. Minor releases never change the internal storage - format and are always compatible with earlier and later minor - releases of the same major version number, e.g., 8.4.2 is compatible - with 8.4, 8.4.1 and 8.4.6. To update between compatible versions, - you simply replace the executables while the server is down and - restart the server. The data directory remains unchanged — - minor upgrades are that simple. + Current PostgreSQL version numbers consist of a + major and a minor version number. For example, in the version number 10.1, + the 10 is the major version number and the 1 is the minor version number, + meaning this would be the first minor release of the major release 10. For + releases before PostgreSQL version 10.0, version + numbers consist of three numbers, for example, 9.5.3. In those cases, the + major version consists of the first two digit groups of the version number, + e.g., 9.5, and the minor version is the third number, e.g., 3, meaning this + would be the third minor release of the major release 9.5. + + + + Minor releases never change the internal storage format and are always + compatible with earlier and later minor releases of the same major version + number. For example, version 10.1 is compatible with version 10.0 and + version 10.6. Similarly, for example, 9.5.3 is compatible with 9.5.0, + 9.5.1, and 9.5.6. To update between compatible versions, you simply + replace the executables while the server is down and restart the server. + The data directory remains unchanged — minor upgrades are that + simple. diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl index 54a14e5dc3..26d165203d 100644 --- a/src/backend/catalog/genbki.pl +++ b/src/backend/catalog/genbki.pl @@ -43,8 +43,8 @@ elsif ($arg =~ /^--set-version=(.*)$/) { $major_version = $1; - die "Version must be in format nn.nn.\n" - if !($major_version =~ /^\d+\.\d+$/); + die "Invalid version string.\n" + if !($major_version =~ /^\d+$/); } else { diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index d4625a6238..22b046e006 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -1334,16 +1334,13 @@ ValidatePgVersion(const char *path) char full_path[MAXPGPATH]; FILE *file; int ret; - long file_major, - file_minor; - long my_major = 0, - my_minor = 0; + long file_major; + long my_major; char *endptr; - const char *version_string = PG_VERSION; + char file_version_string[64]; + const char *my_version_string = PG_VERSION; - my_major = strtol(version_string, &endptr, 10); - if (*endptr == '.') - my_minor = strtol(endptr + 1, NULL, 10); + my_major = strtol(my_version_string, &endptr, 10); snprintf(full_path, sizeof(full_path), "%s/PG_VERSION", path); @@ -1362,8 +1359,11 @@ ValidatePgVersion(const char *path) errmsg("could not open file \"%s\": %m", full_path))); } - ret = fscanf(file, "%ld.%ld", &file_major, &file_minor); - if (ret != 2) + file_version_string[0] = '\0'; + ret = fscanf(file, "%63s", file_version_string); + file_major = strtol(file_version_string, &endptr, 10); + + if (ret != 1 || endptr == file_version_string) ereport(FATAL, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("\"%s\" is not a valid data directory", @@ -1374,13 +1374,13 @@ ValidatePgVersion(const char *path) FreeFile(file); - if (my_major != file_major || my_minor != file_minor) + if (my_major != file_major) ereport(FATAL, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("database files are incompatible with server"), - errdetail("The data directory was initialized by PostgreSQL version %ld.%ld, " + errdetail("The data directory was initialized by PostgreSQL version %s, " "which is not compatible with this version %s.", - file_major, file_minor, version_string))); + file_version_string, my_version_string))); } /*------------------------------------------------------------------------- diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index f901e3c512..ed41dee6a5 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -1082,8 +1082,8 @@ get_bin_version(ClusterInfo *cluster) char cmd[MAXPGPATH], cmd_output[MAX_STRING]; FILE *output; - int pre_dot, - post_dot; + int pre_dot = 0, + post_dot = 0; snprintf(cmd, sizeof(cmd), "\"%s/pg_ctl\" --version", cluster->bindir); @@ -1098,7 +1098,7 @@ get_bin_version(ClusterInfo *cluster) if (strchr(cmd_output, '\n') != NULL) *strchr(cmd_output, '\n') = '\0'; - if (sscanf(cmd_output, "%*s %*s %d.%d", &pre_dot, &post_dot) != 2) + if (sscanf(cmd_output, "%*s %*s %d.%d", &pre_dot, &post_dot) < 1) pg_fatal("could not get version from %s\n", cmd); cluster->bin_version = (pre_dot * 100 + post_dot) * 100; diff --git a/src/bin/pg_upgrade/server.c b/src/bin/pg_upgrade/server.c index 830335f501..12432bb1d0 100644 --- a/src/bin/pg_upgrade/server.c +++ b/src/bin/pg_upgrade/server.c @@ -166,7 +166,7 @@ get_major_server_version(ClusterInfo *cluster) if (fscanf(version_fd, "%63s", cluster->major_version_str) == 0 || sscanf(cluster->major_version_str, "%d.%d", &integer_version, - &fractional_version) != 2) + &fractional_version) < 1) pg_fatal("could not get version from %s\n", cluster->pgdata); fclose(version_fd); diff --git a/src/include/pg_config.h.win32 b/src/include/pg_config.h.win32 index b6b88fcf0d..8892c3cb4f 100644 --- a/src/include/pg_config.h.win32 +++ b/src/include/pg_config.h.win32 @@ -554,10 +554,10 @@ #define PACKAGE_NAME "PostgreSQL" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "PostgreSQL 9.6beta4" +#define PACKAGE_STRING "PostgreSQL 10devel" /* Define to the version of this package. */ -#define PACKAGE_VERSION "9.6beta4" +#define PACKAGE_VERSION "10devel" /* Define to the name of a signed 128-bit integer type. */ #undef PG_INT128_TYPE @@ -566,10 +566,10 @@ #define PG_INT64_TYPE long long int /* PostgreSQL version as a string */ -#define PG_VERSION "9.6beta4" +#define PG_VERSION "10devel" /* PostgreSQL version as a number */ -#define PG_VERSION_NUM 90600 +#define PG_VERSION_NUM 100000 /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "postgresql" diff --git a/src/interfaces/libpq/libpq.rc.in b/src/interfaces/libpq/libpq.rc.in index e41a1a27f4..0d6f7049f5 100644 --- a/src/interfaces/libpq/libpq.rc.in +++ b/src/interfaces/libpq/libpq.rc.in @@ -1,8 +1,8 @@ #include VS_VERSION_INFO VERSIONINFO - FILEVERSION 9,6,0,0 - PRODUCTVERSION 9,6,0,0 + FILEVERSION 10,0,0,0 + PRODUCTVERSION 10,0,0,0 FILEFLAGSMASK 0x3fL FILEFLAGS 0 FILEOS VOS__WINDOWS32 @@ -15,13 +15,13 @@ BEGIN BEGIN VALUE "CompanyName", "\0" VALUE "FileDescription", "PostgreSQL Access Library\0" - VALUE "FileVersion", "9.6.0\0" + VALUE "FileVersion", "10.0\0" VALUE "InternalName", "libpq\0" VALUE "LegalCopyright", "Copyright (C) 2016\0" VALUE "LegalTrademarks", "\0" VALUE "OriginalFilename", "libpq.dll\0" VALUE "ProductName", "PostgreSQL\0" - VALUE "ProductVersion", "9.6.0\0" + VALUE "ProductVersion", "10.0\0" END END BLOCK "VarFileInfo" diff --git a/src/port/win32ver.rc b/src/port/win32ver.rc index c21b74c017..3ce092382b 100644 --- a/src/port/win32ver.rc +++ b/src/port/win32ver.rc @@ -2,8 +2,8 @@ #include "pg_config.h" VS_VERSION_INFO VERSIONINFO - FILEVERSION 9,6,0,0 - PRODUCTVERSION 9,6,0,0 + FILEVERSION 10,0,0,0 + PRODUCTVERSION 10,0,0,0 FILEFLAGSMASK 0x17L FILEFLAGS 0x0L FILEOS VOS_NT_WINDOWS32 diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index f07029bce1..9cb1ad36cf 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -131,12 +131,12 @@ sub GenerateFiles if (/^AC_INIT\(\[PostgreSQL\], \[([^\]]+)\]/) { $self->{strver} = $1; - if ($self->{strver} !~ /^(\d+)\.(\d+)(?:\.(\d+))?/) + if ($self->{strver} !~ /^(\d+)(?:\.(\d+))?/) { confess "Bad format of version: $self->{strver}\n"; } - $self->{numver} = sprintf("%d%02d%02d", $1, $2, $3 ? $3 : 0); - $self->{majorver} = sprintf("%d.%d", $1, $2); + $self->{numver} = sprintf("%d%04d", $1, $2 ? $2 : 0); + $self->{majorver} = sprintf("%d", $1); } } close(C); diff --git a/src/tools/version_stamp.pl b/src/tools/version_stamp.pl index cc685453dd..3edd7bedaf 100755 --- a/src/tools/version_stamp.pl +++ b/src/tools/version_stamp.pl @@ -22,8 +22,7 @@ # Major version is hard-wired into the script. We update it when we branch # a new development version. -$major1 = 9; -$major2 = 6; +$majorversion = 10; # Validate argument and compute derived variables $minor = shift; @@ -60,7 +59,6 @@ } # Create various required forms of the version number -$majorversion = $major1 . "." . $major2; if ($dotneeded) { $fullversion = $majorversion . "." . $minor; @@ -70,7 +68,7 @@ $fullversion = $majorversion . $minor; } $numericversion = $majorversion . "." . $numericminor; -$padnumericversion = sprintf("%d%02d%02d", $major1, $major2, $numericminor); +$padnumericversion = sprintf("%d%04d", $majorversion, $numericminor); # Get the autoconf version number for eventual nag message # (this also ensures we're in the right directory) @@ -110,15 +108,15 @@ ); sed_file("src/interfaces/libpq/libpq.rc.in", -"-e 's/FILEVERSION [0-9]*,[0-9]*,[0-9]*,0/FILEVERSION $major1,$major2,$numericminor,0/' " - . "-e 's/PRODUCTVERSION [0-9]*,[0-9]*,[0-9]*,0/PRODUCTVERSION $major1,$major2,$numericminor,0/' " +"-e 's/FILEVERSION [0-9]*,[0-9]*,[0-9]*,0/FILEVERSION $majorversion,0,$numericminor,0/' " + . "-e 's/PRODUCTVERSION [0-9]*,[0-9]*,[0-9]*,0/PRODUCTVERSION $majorversion,0,$numericminor,0/' " . "-e 's/VALUE \"FileVersion\", \"[0-9.]*/VALUE \"FileVersion\", \"$numericversion/' " . "-e 's/VALUE \"ProductVersion\", \"[0-9.]*/VALUE \"ProductVersion\", \"$numericversion/'" ); sed_file("src/port/win32ver.rc", -"-e 's/FILEVERSION [0-9]*,[0-9]*,[0-9]*,0/FILEVERSION $major1,$major2,$numericminor,0/' " - . "-e 's/PRODUCTVERSION [0-9]*,[0-9]*,[0-9]*,0/PRODUCTVERSION $major1,$major2,$numericminor,0/'" +"-e 's/FILEVERSION [0-9]*,[0-9]*,[0-9]*,0/FILEVERSION $majorversion,0,$numericminor,0/' " + . "-e 's/PRODUCTVERSION [0-9]*,[0-9]*,[0-9]*,0/PRODUCTVERSION $majorversion,0,$numericminor,0/'" ); print "Stamped these files with version number $fullversion:\n$fixedfiles"; From 0b9358d4406b9b45a06855d53f491cc7ce9550a9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 15 Aug 2016 14:35:55 -0400 Subject: [PATCH 002/871] Stamp shared-library minor version numbers for v10. --- src/interfaces/ecpg/compatlib/Makefile | 2 +- src/interfaces/ecpg/ecpglib/Makefile | 2 +- src/interfaces/ecpg/pgtypeslib/Makefile | 2 +- src/interfaces/ecpg/preproc/Makefile | 2 +- src/interfaces/libpq/Makefile | 2 +- src/tools/msvc/Mkvcbuild.pm | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/interfaces/ecpg/compatlib/Makefile b/src/interfaces/ecpg/compatlib/Makefile index 0a04a7310f..2f58ad8592 100644 --- a/src/interfaces/ecpg/compatlib/Makefile +++ b/src/interfaces/ecpg/compatlib/Makefile @@ -16,7 +16,7 @@ include $(top_builddir)/src/Makefile.global PGFILEDESC = "ECPG compat - compatibility library for ECPG" NAME= ecpg_compat SO_MAJOR_VERSION= 3 -SO_MINOR_VERSION= 8 +SO_MINOR_VERSION= 9 override CPPFLAGS := -I../include -I$(top_srcdir)/src/interfaces/ecpg/include \ -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS) diff --git a/src/interfaces/ecpg/ecpglib/Makefile b/src/interfaces/ecpg/ecpglib/Makefile index 39c4232580..00503b33dd 100644 --- a/src/interfaces/ecpg/ecpglib/Makefile +++ b/src/interfaces/ecpg/ecpglib/Makefile @@ -16,7 +16,7 @@ include $(top_builddir)/src/Makefile.global PGFILEDESC = "ECPG - embedded SQL in C" NAME= ecpg SO_MAJOR_VERSION= 6 -SO_MINOR_VERSION= 8 +SO_MINOR_VERSION= 9 override CPPFLAGS := -I../include -I$(top_srcdir)/src/interfaces/ecpg/include \ -I$(libpq_srcdir) -I$(top_builddir)/src/port -DFRONTEND $(CPPFLAGS) diff --git a/src/interfaces/ecpg/pgtypeslib/Makefile b/src/interfaces/ecpg/pgtypeslib/Makefile index 1c1a42fa8f..ac278948a5 100644 --- a/src/interfaces/ecpg/pgtypeslib/Makefile +++ b/src/interfaces/ecpg/pgtypeslib/Makefile @@ -16,7 +16,7 @@ include $(top_builddir)/src/Makefile.global PGFILEDESC = "pgtypes - library for data type mapping" NAME= pgtypes SO_MAJOR_VERSION= 3 -SO_MINOR_VERSION= 7 +SO_MINOR_VERSION= 8 override CPPFLAGS := -I../include -I$(top_srcdir)/src/interfaces/ecpg/include \ -DFRONTEND $(CPPFLAGS) diff --git a/src/interfaces/ecpg/preproc/Makefile b/src/interfaces/ecpg/preproc/Makefile index 30db5a049a..7dd4b2ffdb 100644 --- a/src/interfaces/ecpg/preproc/Makefile +++ b/src/interfaces/ecpg/preproc/Makefile @@ -16,7 +16,7 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global MAJOR_VERSION= 4 -MINOR_VERSION= 12 +MINOR_VERSION= 13 PATCHLEVEL=0 override CPPFLAGS := -I../include -I$(top_srcdir)/src/interfaces/ecpg/include \ diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 1b292d2cf2..83b30b0f9e 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -17,7 +17,7 @@ include $(top_builddir)/src/Makefile.global # shared library parameters NAME= pq SO_MAJOR_VERSION= 5 -SO_MINOR_VERSION= 9 +SO_MINOR_VERSION= 10 override CPPFLAGS := -DFRONTEND -DUNSAFE_STAT_OK -I$(srcdir) $(CPPFLAGS) -I$(top_builddir)/src/port -I$(top_srcdir)/src/port ifneq ($(PORTNAME), win32) diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index fe905d3c9d..16180f68ed 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -270,7 +270,7 @@ sub mkvcbuild $ecpg->AddPrefixInclude('src/interfaces/ecpg/preproc'); $ecpg->AddFiles('src/interfaces/ecpg/preproc', 'pgc.l', 'preproc.y'); $ecpg->AddDefine('MAJOR_VERSION=4'); - $ecpg->AddDefine('MINOR_VERSION=12'); + $ecpg->AddDefine('MINOR_VERSION=13'); $ecpg->AddDefine('PATCHLEVEL=0'); $ecpg->AddDefine('ECPG_COMPILE'); $ecpg->AddReference($libpgcommon, $libpgport); From 3149a12166120d0b476f5ca7837ebcf0e7124703 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 15 Aug 2016 15:24:54 -0400 Subject: [PATCH 003/871] Update git_changelog to know that there's a 9.6 branch. Missed this in the main 10devel version stamping patch. --- src/tools/git_changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/git_changelog b/src/tools/git_changelog index c9a503f3fe..b9e631bb9c 100755 --- a/src/tools/git_changelog +++ b/src/tools/git_changelog @@ -57,7 +57,7 @@ require IPC::Open2; # (We could get this from "git branches", but not worth the trouble.) # NB: master must be first! my @BRANCHES = qw(master - REL9_5_STABLE + REL9_6_STABLE REL9_5_STABLE REL9_4_STABLE REL9_3_STABLE REL9_2_STABLE REL9_1_STABLE REL9_0_STABLE REL8_4_STABLE REL8_3_STABLE REL8_2_STABLE REL8_1_STABLE REL8_0_STABLE REL7_4_STABLE REL7_3_STABLE REL7_2_STABLE REL7_1_STABLE REL7_0_PATCHES From 2bf06f756142f4c398270cdc30bbba9b9dfbd38b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 15 Aug 2016 17:35:35 -0400 Subject: [PATCH 004/871] Allow .so minor version numbers above 9 in .gitignore. Needed now that libpq.so's minor version has reached 10. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index cbf8d7996a..4976fd9119 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,11 @@ *.so *.so.[0-9] *.so.[0-9].[0-9] +*.so.[0-9].[0-9][0-9] *.sl *.sl.[0-9] *.sl.[0-9].[0-9] +*.sl.[0-9].[0-9][0-9] *.dylib *.dll *.exp From b25b6c9701e5c18e3ad3b701df62380f8d138ba0 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 15 Aug 2016 18:09:55 -0400 Subject: [PATCH 005/871] Once again allow LWLocks to be used within DSM segments. Prior to commit 7882c3b0b95640e361f1533fe0f2d02e4e5d8610, it was possible to use LWLocks within DSM segments, but that commit broke this use case by switching from a doubly linked list to a circular linked list. Switch back, using a new bit of general infrastructure for maintaining lists of PGPROCs. Thomas Munro, reviewed by me. --- src/backend/storage/lmgr/lwlock.c | 63 ++++++----- src/include/storage/lwlock.h | 4 +- src/include/storage/proc.h | 6 +- src/include/storage/proclist.h | 154 +++++++++++++++++++++++++++ src/include/storage/proclist_types.h | 45 ++++++++ src/tools/pgindent/typedefs.list | 3 + 6 files changed, 240 insertions(+), 35 deletions(-) create mode 100644 src/include/storage/proclist.h create mode 100644 src/include/storage/proclist_types.h diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 7ffa87d914..303e99c65b 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -84,6 +84,7 @@ #include "storage/ipc.h" #include "storage/predicate.h" #include "storage/proc.h" +#include "storage/proclist.h" #include "storage/spin.h" #include "utils/memutils.h" @@ -717,7 +718,7 @@ LWLockInitialize(LWLock *lock, int tranche_id) pg_atomic_init_u32(&lock->nwaiters, 0); #endif lock->tranche = tranche_id; - dlist_init(&lock->waiters); + proclist_init(&lock->waiters); } /* @@ -920,25 +921,25 @@ LWLockWakeup(LWLock *lock) { bool new_release_ok; bool wokeup_somebody = false; - dlist_head wakeup; - dlist_mutable_iter iter; + proclist_head wakeup; + proclist_mutable_iter iter; - dlist_init(&wakeup); + proclist_init(&wakeup); new_release_ok = true; /* lock wait list while collecting backends to wake up */ LWLockWaitListLock(lock); - dlist_foreach_modify(iter, &lock->waiters) + proclist_foreach_modify(iter, &lock->waiters, lwWaitLink) { - PGPROC *waiter = dlist_container(PGPROC, lwWaitLink, iter.cur); + PGPROC *waiter = GetPGProcByNumber(iter.cur); if (wokeup_somebody && waiter->lwWaitMode == LW_EXCLUSIVE) continue; - dlist_delete(&waiter->lwWaitLink); - dlist_push_tail(&wakeup, &waiter->lwWaitLink); + proclist_delete(&lock->waiters, iter.cur, lwWaitLink); + proclist_push_tail(&wakeup, iter.cur, lwWaitLink); if (waiter->lwWaitMode != LW_WAIT_UNTIL_FREE) { @@ -963,7 +964,7 @@ LWLockWakeup(LWLock *lock) break; } - Assert(dlist_is_empty(&wakeup) || pg_atomic_read_u32(&lock->state) & LW_FLAG_HAS_WAITERS); + Assert(proclist_is_empty(&wakeup) || pg_atomic_read_u32(&lock->state) & LW_FLAG_HAS_WAITERS); /* unset required flags, and release lock, in one fell swoop */ { @@ -982,7 +983,7 @@ LWLockWakeup(LWLock *lock) else desired_state &= ~LW_FLAG_RELEASE_OK; - if (dlist_is_empty(&wakeup)) + if (proclist_is_empty(&wakeup)) desired_state &= ~LW_FLAG_HAS_WAITERS; desired_state &= ~LW_FLAG_LOCKED; /* release lock */ @@ -994,12 +995,12 @@ LWLockWakeup(LWLock *lock) } /* Awaken any waiters I removed from the queue. */ - dlist_foreach_modify(iter, &wakeup) + proclist_foreach_modify(iter, &wakeup, lwWaitLink) { - PGPROC *waiter = dlist_container(PGPROC, lwWaitLink, iter.cur); + PGPROC *waiter = GetPGProcByNumber(iter.cur); LOG_LWDEBUG("LWLockRelease", lock, "release waiter"); - dlist_delete(&waiter->lwWaitLink); + proclist_delete(&wakeup, iter.cur, lwWaitLink); /* * Guarantee that lwWaiting being unset only becomes visible once the @@ -1046,9 +1047,9 @@ LWLockQueueSelf(LWLock *lock, LWLockMode mode) /* LW_WAIT_UNTIL_FREE waiters are always at the front of the queue */ if (mode == LW_WAIT_UNTIL_FREE) - dlist_push_head(&lock->waiters, &MyProc->lwWaitLink); + proclist_push_head(&lock->waiters, MyProc->pgprocno, lwWaitLink); else - dlist_push_tail(&lock->waiters, &MyProc->lwWaitLink); + proclist_push_tail(&lock->waiters, MyProc->pgprocno, lwWaitLink); /* Can release the mutex now */ LWLockWaitListUnlock(lock); @@ -1070,7 +1071,7 @@ static void LWLockDequeueSelf(LWLock *lock) { bool found = false; - dlist_mutable_iter iter; + proclist_mutable_iter iter; #ifdef LWLOCK_STATS lwlock_stats *lwstats; @@ -1086,19 +1087,17 @@ LWLockDequeueSelf(LWLock *lock) * Can't just remove ourselves from the list, but we need to iterate over * all entries as somebody else could have unqueued us. */ - dlist_foreach_modify(iter, &lock->waiters) + proclist_foreach_modify(iter, &lock->waiters, lwWaitLink) { - PGPROC *proc = dlist_container(PGPROC, lwWaitLink, iter.cur); - - if (proc == MyProc) + if (iter.cur == MyProc->pgprocno) { found = true; - dlist_delete(&proc->lwWaitLink); + proclist_delete(&lock->waiters, iter.cur, lwWaitLink); break; } } - if (dlist_is_empty(&lock->waiters) && + if (proclist_is_empty(&lock->waiters) && (pg_atomic_read_u32(&lock->state) & LW_FLAG_HAS_WAITERS) != 0) { pg_atomic_fetch_and_u32(&lock->state, ~LW_FLAG_HAS_WAITERS); @@ -1719,12 +1718,12 @@ LWLockWaitForVar(LWLock *lock, uint64 *valptr, uint64 oldval, uint64 *newval) void LWLockUpdateVar(LWLock *lock, uint64 *valptr, uint64 val) { - dlist_head wakeup; - dlist_mutable_iter iter; + proclist_head wakeup; + proclist_mutable_iter iter; PRINT_LWDEBUG("LWLockUpdateVar", lock, LW_EXCLUSIVE); - dlist_init(&wakeup); + proclist_init(&wakeup); LWLockWaitListLock(lock); @@ -1737,15 +1736,15 @@ LWLockUpdateVar(LWLock *lock, uint64 *valptr, uint64 val) * See if there are any LW_WAIT_UNTIL_FREE waiters that need to be woken * up. They are always in the front of the queue. */ - dlist_foreach_modify(iter, &lock->waiters) + proclist_foreach_modify(iter, &lock->waiters, lwWaitLink) { - PGPROC *waiter = dlist_container(PGPROC, lwWaitLink, iter.cur); + PGPROC *waiter = GetPGProcByNumber(iter.cur); if (waiter->lwWaitMode != LW_WAIT_UNTIL_FREE) break; - dlist_delete(&waiter->lwWaitLink); - dlist_push_tail(&wakeup, &waiter->lwWaitLink); + proclist_delete(&lock->waiters, iter.cur, lwWaitLink); + proclist_push_tail(&wakeup, iter.cur, lwWaitLink); } /* We are done updating shared state of the lock itself. */ @@ -1754,11 +1753,11 @@ LWLockUpdateVar(LWLock *lock, uint64 *valptr, uint64 val) /* * Awaken any waiters I removed from the queue. */ - dlist_foreach_modify(iter, &wakeup) + proclist_foreach_modify(iter, &wakeup, lwWaitLink) { - PGPROC *waiter = dlist_container(PGPROC, lwWaitLink, iter.cur); + PGPROC *waiter = GetPGProcByNumber(iter.cur); - dlist_delete(&waiter->lwWaitLink); + proclist_delete(&wakeup, iter.cur, lwWaitLink); /* check comment in LWLockWakeup() about this barrier */ pg_write_barrier(); waiter->lwWaiting = false; diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 3db11e43f0..959f5f1e4d 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -18,7 +18,7 @@ #error "lwlock.h may not be included from frontend code" #endif -#include "lib/ilist.h" +#include "storage/proclist_types.h" #include "storage/s_lock.h" #include "port/atomics.h" @@ -59,7 +59,7 @@ typedef struct LWLock { uint16 tranche; /* tranche ID */ pg_atomic_uint32 state; /* state of exclusive/nonexclusive lockers */ - dlist_head waiters; /* list of waiting PGPROCs */ + proclist_head waiters; /* list of waiting PGPROCs */ #ifdef LOCK_DEBUG pg_atomic_uint32 nwaiters; /* number of waiters */ struct PGPROC *owner; /* last exclusive owner of the lock */ diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index 775c66a197..f576f052df 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -19,6 +19,7 @@ #include "storage/latch.h" #include "storage/lock.h" #include "storage/pg_sema.h" +#include "storage/proclist_types.h" /* * Each backend advertises up to PGPROC_MAX_CACHED_SUBXIDS TransactionIds @@ -112,7 +113,7 @@ struct PGPROC /* Info about LWLock the process is currently waiting for, if any. */ bool lwWaiting; /* true if waiting for an LW lock */ uint8 lwWaitMode; /* lwlock mode being waited for */ - dlist_node lwWaitLink; /* position in LW lock wait list */ + proclist_node lwWaitLink; /* position in LW lock wait list */ /* Info about lock the process is currently waiting for, if any. */ /* waitLock and waitProcLock are NULL if not currently waiting. */ @@ -243,6 +244,9 @@ extern PROC_HDR *ProcGlobal; extern PGPROC *PreparedXactProcs; +/* Accessor for PGPROC given a pgprocno. */ +#define GetPGProcByNumber(n) (&ProcGlobal->allProcs[(n)]) + /* * We set aside some extra PGPROC structures for auxiliary processes, * ie things that aren't full-fledged backends but need shmem access. diff --git a/src/include/storage/proclist.h b/src/include/storage/proclist.h new file mode 100644 index 0000000000..2013a406a3 --- /dev/null +++ b/src/include/storage/proclist.h @@ -0,0 +1,154 @@ +/*------------------------------------------------------------------------- + * + * proclist.h + * operations on doubly-linked lists of pgprocnos + * + * The interface is similar to dlist from ilist.h, but uses pgprocno instead + * of pointers. This allows proclist_head to be mapped at different addresses + * in different backends. + * + * See proclist_types.h for the structs that these functions operate on. They + * are separated to break a header dependency cycle with proc.h. + * + * Portions Copyright (c) 2016, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/include/storage/proclist.h + *------------------------------------------------------------------------- + */ +#ifndef PROCLIST_H +#define PROCLIST_H + +#include "storage/proc.h" +#include "storage/proclist_types.h" + +/* + * Initialize a proclist. + */ +static inline void +proclist_init(proclist_head *list) +{ + list->head = list->tail = INVALID_PGPROCNO; +} + +/* + * Is the list empty? + */ +static inline bool +proclist_is_empty(proclist_head *list) +{ + return list->head == INVALID_PGPROCNO; +} + +/* + * Get a pointer to a proclist_node inside a given PGPROC, given a procno and + * an offset. + */ +static inline proclist_node * +proclist_node_get(int procno, size_t node_offset) +{ + char *entry = (char *) GetPGProcByNumber(procno); + + return (proclist_node *) (entry + node_offset); +} + +/* + * Insert a node at the beginning of a list. + */ +static inline void +proclist_push_head_offset(proclist_head *list, int procno, size_t node_offset) +{ + proclist_node *node = proclist_node_get(procno, node_offset); + + if (list->head == INVALID_PGPROCNO) + { + Assert(list->tail == INVALID_PGPROCNO); + node->next = node->prev = INVALID_PGPROCNO; + list->head = list->tail = procno; + } + else + { + Assert(list->tail != INVALID_PGPROCNO); + node->next = list->head; + proclist_node_get(node->next, node_offset)->prev = procno; + node->prev = INVALID_PGPROCNO; + list->head = procno; + } +} + +/* + * Insert a node a the end of a list. + */ +static inline void +proclist_push_tail_offset(proclist_head *list, int procno, size_t node_offset) +{ + proclist_node *node = proclist_node_get(procno, node_offset); + + if (list->tail == INVALID_PGPROCNO) + { + Assert(list->head == INVALID_PGPROCNO); + node->next = node->prev = INVALID_PGPROCNO; + list->head = list->tail = procno; + } + else + { + Assert(list->head != INVALID_PGPROCNO); + node->prev = list->tail; + proclist_node_get(node->prev, node_offset)->next = procno; + node->next = INVALID_PGPROCNO; + list->tail = procno; + } +} + +/* + * Delete a node. The node must be in the list. + */ +static inline void +proclist_delete_offset(proclist_head *list, int procno, size_t node_offset) +{ + proclist_node *node = proclist_node_get(procno, node_offset); + + if (node->prev == INVALID_PGPROCNO) + list->head = node->next; + else + proclist_node_get(node->prev, node_offset)->next = node->next; + + if (node->next == INVALID_PGPROCNO) + list->tail = node->prev; + else + proclist_node_get(node->next, node_offset)->prev = node->prev; +} + +/* + * Helper macros to avoid repetition of offsetof(PGPROC, ). + * 'link_member' is the name of a proclist_node member in PGPROC. + */ +#define proclist_delete(list, procno, link_member) \ + proclist_delete_offset((list), (procno), offsetof(PGPROC, link_member)) +#define proclist_push_head(list, procno, link_member) \ + proclist_push_head_offset((list), (procno), offsetof(PGPROC, link_member)) +#define proclist_push_tail(list, procno, link_member) \ + proclist_push_tail_offset((list), (procno), offsetof(PGPROC, link_member)) + +/* + * Iterate through the list pointed at by 'lhead', storing the current + * position in 'iter'. 'link_member' is the name of a proclist_node member in + * PGPROC. Access the current position with iter.cur. + * + * The only list modification allowed while iterating is deleting the current + * node with proclist_delete(list, iter.cur, node_offset). + */ +#define proclist_foreach_modify(iter, lhead, link_member) \ + for (AssertVariableIsOfTypeMacro(iter, proclist_mutable_iter), \ + AssertVariableIsOfTypeMacro(lhead, proclist_head *), \ + (iter).cur = (lhead)->head, \ + (iter).next = (iter).cur == INVALID_PGPROCNO ? INVALID_PGPROCNO : \ + proclist_node_get((iter).cur, \ + offsetof(PGPROC, link_member))->next; \ + (iter).cur != INVALID_PGPROCNO; \ + (iter).cur = (iter).next, \ + (iter).next = (iter).cur == INVALID_PGPROCNO ? INVALID_PGPROCNO : \ + proclist_node_get((iter).cur, \ + offsetof(PGPROC, link_member))->next) + +#endif diff --git a/src/include/storage/proclist_types.h b/src/include/storage/proclist_types.h new file mode 100644 index 0000000000..b8b0326853 --- /dev/null +++ b/src/include/storage/proclist_types.h @@ -0,0 +1,45 @@ +/*------------------------------------------------------------------------- + * + * proclist_types.h + * doubly-linked lists of pgprocnos + * + * See proclist.h for functions that operate on these types. + * + * Portions Copyright (c) 2016, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/include/storage/proclist_types.h + *------------------------------------------------------------------------- + */ + +#ifndef PROCLIST_TYPES_H +#define PROCLIST_TYPES_H + +/* + * A node in a list of processes. + */ +typedef struct proclist_node +{ + int next; /* pgprocno of the next PGPROC */ + int prev; /* pgprocno of the prev PGPROC */ +} proclist_node; + +/* + * Head of a doubly-linked list of PGPROCs, identified by pgprocno. + */ +typedef struct proclist_head +{ + int head; /* pgprocno of the head PGPROC */ + int tail; /* pgprocno of the tail PGPROC */ +} proclist_head; + +/* + * List iterator allowing some modifications while iterating. + */ +typedef struct proclist_mutable_iter +{ + int cur; /* pgprocno of the current PGPROC */ + int next; /* pgprocno of the next PGPROC */ +} proclist_mutable_iter; + +#endif diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index dff6f65ef0..932be2f665 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2668,6 +2668,9 @@ printTextRule priv_map process_file_callback_t process_sublinks_context +proclist_head +proclist_mutable_iter +proclist_node promptStatus_t pthread_attr_t pthread_key_t From 8fc571b7dd9fa1659536a26bb085584b50a65a51 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 16 Aug 2016 10:59:14 -0400 Subject: [PATCH 006/871] Doc: remove out-of-date claim that pg_am rows must be inserted by hand. Commit 473b93287 added a sentence about that, but neglected to remove the adjacent sentence it had falsified. Per Alexander Law. --- doc/src/sgml/indexam.sgml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index b59cd0363a..40f201b11b 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -51,16 +51,9 @@ pg_am system catalog. The pg_am entry specifies a name and a handler function for the access - method. There is not currently any special support - for creating or deleting pg_am entries; - anyone able to write a new access method is expected to be competent - to insert an appropriate row for themselves. - - - - Index access methods can be defined and dropped using + method. These entries can be created and deleted using the and - SQL commands respectively. + SQL commands. From 9b002cc9fec557fcfe17d67f55b53804447230e5 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 16 Aug 2016 11:35:36 -0400 Subject: [PATCH 007/871] Doc: copy-editing in create_access_method.sgml. Improve shaky English grammar. And markup. --- doc/src/sgml/ref/create_access_method.sgml | 27 +++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/doc/src/sgml/ref/create_access_method.sgml b/doc/src/sgml/ref/create_access_method.sgml index 3c091f8021..0a30e6ea3c 100644 --- a/doc/src/sgml/ref/create_access_method.sgml +++ b/doc/src/sgml/ref/create_access_method.sgml @@ -57,29 +57,28 @@ CREATE ACCESS METHOD name - access_method_type + access_method_type - This clause specifies type of access method to define. + This clause specifies the type of access method to define. Only INDEX is supported at present. - HANDLER handler_function + handler_function - handler_function is the - name of a previously registered function that will be called to - retrieve the struct which contains required parameters and functions - of access method to the core. The handler function must take single - argument of type internal, and its return type depends on the - type of access method; for INDEX access methods, it - must be index_am_handler. - - - See for index access methods API. + handler_function is the + name (possibly schema-qualified) of a previously registered function + that represents the access method. The handler function must be + declared to take a single argument of type internal, + and its return type depends on the type of access method; + for INDEX access methods, it must + be index_am_handler. The C-level API that the handler + function must implement varies depending on the type of access method. + The index access method API is described in . @@ -90,7 +89,7 @@ CREATE ACCESS METHOD name Examples - Create an access method heptree with + Create an index access method heptree with handler function heptree_handler: CREATE ACCESS METHOD heptree TYPE INDEX HANDLER heptree_handler; From a7b5573d665c8a37fad9bc69f44c5b4e8760a73b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 16 Aug 2016 12:49:30 -0400 Subject: [PATCH 008/871] Remove separate version numbering for ecpg preprocessor. Once upon a time, it made sense for the ecpg preprocessor to have its own version number, because it used a manually-maintained grammar that wasn't always in sync with the core grammar. But those days are thankfully long gone, leaving only a maintenance nuisance behind. Let's use the PG v10 version numbering changeover as an excuse to get rid of the ecpg version number and just have ecpg identify itself by PG_VERSION. From the user's standpoint, ecpg will go from "4.12" in the 9.6 branch to "10" in the 10 branch, so there's no failure of monotonicity. Discussion: <1471332659.4410.67.camel@postgresql.org> --- src/interfaces/ecpg/preproc/Makefile | 9 ++------- src/interfaces/ecpg/preproc/ecpg.c | 10 +++++----- src/tools/RELEASE_CHANGES | 4 +--- src/tools/msvc/Mkvcbuild.pm | 3 --- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/src/interfaces/ecpg/preproc/Makefile b/src/interfaces/ecpg/preproc/Makefile index 7dd4b2ffdb..b5dbdd6150 100644 --- a/src/interfaces/ecpg/preproc/Makefile +++ b/src/interfaces/ecpg/preproc/Makefile @@ -15,16 +15,11 @@ subdir = src/interfaces/ecpg/preproc top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -MAJOR_VERSION= 4 -MINOR_VERSION= 13 -PATCHLEVEL=0 - override CPPFLAGS := -I../include -I$(top_srcdir)/src/interfaces/ecpg/include \ - -I. -I$(srcdir) -DMAJOR_VERSION=$(MAJOR_VERSION) \ - -DMINOR_VERSION=$(MINOR_VERSION) -DPATCHLEVEL=$(PATCHLEVEL) \ + -I. -I$(srcdir) -DECPG_COMPILE \ $(CPPFLAGS) -override CFLAGS += $(PTHREAD_CFLAGS) -DECPG_COMPILE +override CFLAGS += $(PTHREAD_CFLAGS) OBJS= preproc.o pgc.o type.o ecpg.o output.o parser.o \ keywords.o c_keywords.o ecpg_keywords.o ../ecpglib/typename.o descriptor.o variable.o \ diff --git a/src/interfaces/ecpg/preproc/ecpg.c b/src/interfaces/ecpg/preproc/ecpg.c index c7fd034bc1..3ce9d04bcc 100644 --- a/src/interfaces/ecpg/preproc/ecpg.c +++ b/src/interfaces/ecpg/preproc/ecpg.c @@ -150,8 +150,7 @@ main(int argc, char *const argv[]) switch (c) { case ECPG_GETOPT_LONG_VERSION: - printf("ecpg (PostgreSQL %s) %d.%d.%d\n", PG_VERSION, - MAJOR_VERSION, MINOR_VERSION, PATCHLEVEL); + printf("ecpg %s\n", PG_VERSION); exit(0); case ECPG_GETOPT_LONG_HELP: help(progname); @@ -264,8 +263,9 @@ main(int argc, char *const argv[]) if (verbose) { - fprintf(stderr, _("%s, the PostgreSQL embedded C preprocessor, version %d.%d.%d\n"), - progname, MAJOR_VERSION, MINOR_VERSION, PATCHLEVEL); + fprintf(stderr, + _("%s, the PostgreSQL embedded C preprocessor, version %s\n"), + progname, PG_VERSION); fprintf(stderr, _("EXEC SQL INCLUDE ... search starts here:\n")); for (ip = include_paths; ip != NULL; ip = ip->next) fprintf(stderr, " %s\n", ip->path); @@ -440,7 +440,7 @@ main(int argc, char *const argv[]) if (regression_mode) fprintf(yyout, "/* Processed by ecpg (regression mode) */\n"); else - fprintf(yyout, "/* Processed by ecpg (%d.%d.%d) */\n", MAJOR_VERSION, MINOR_VERSION, PATCHLEVEL); + fprintf(yyout, "/* Processed by ecpg (%s) */\n", PG_VERSION); if (header_mode == false) { diff --git a/src/tools/RELEASE_CHANGES b/src/tools/RELEASE_CHANGES index ad49220592..e6e294b07e 100644 --- a/src/tools/RELEASE_CHANGES +++ b/src/tools/RELEASE_CHANGES @@ -80,13 +80,11 @@ Starting a New Development Cycle * Add version tag to src/tools/git_changelog * Bump minor library versions, major if appropriate (see below) - o Look for SO_MINOR_VERSION and MINOR_VERSION macros in + o Look for SO_MINOR_VERSION macros in src/interfaces/ecpg/compatlib/Makefile src/interfaces/ecpg/ecpglib/Makefile src/interfaces/ecpg/pgtypeslib/Makefile - src/interfaces/ecpg/preproc/Makefile src/interfaces/libpq/Makefile - src/tools/msvc/Mkvcbuild.pm Creating Back-Branch Release Notes diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 16180f68ed..da4d9847fc 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -269,9 +269,6 @@ sub mkvcbuild $ecpg->AddIncludeDir('src/interfaces/libpq'); $ecpg->AddPrefixInclude('src/interfaces/ecpg/preproc'); $ecpg->AddFiles('src/interfaces/ecpg/preproc', 'pgc.l', 'preproc.y'); - $ecpg->AddDefine('MAJOR_VERSION=4'); - $ecpg->AddDefine('MINOR_VERSION=13'); - $ecpg->AddDefine('PATCHLEVEL=0'); $ecpg->AddDefine('ECPG_COMPILE'); $ecpg->AddReference($libpgcommon, $libpgport); From 41fb35fabf03bffa812caddf24323d4d06f811ba Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 16 Aug 2016 13:23:32 -0400 Subject: [PATCH 009/871] Fix possible crash due to incorrect allocation context. Commit af33039317ddc4a0e38a02e2255c2bf453115fd2 aimed to reduce leakage from tqueue.c, which is good. Unfortunately, by changing the memory context in which all of gather_readnext() executes, it also changed the context in which ExecShutdownGatherWorkers executes, which is not good, because that function eventually causes a call to ExecParallelRetrieveInstrumentation, which proceeds to allocate planstate->worker_instrument in a short-lived context, causing a crash. Rushabh Lathia, reviewed by Amit Kapila and by me. --- src/backend/executor/execParallel.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index 380d743f6c..5aa6f023bf 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -500,6 +500,7 @@ ExecParallelRetrieveInstrumentation(PlanState *planstate, int n; int ibytes; int plan_node_id = planstate->plan->plan_node_id; + MemoryContext oldcontext; /* Find the instumentation for this node. */ for (i = 0; i < instrumentation->num_plan_nodes; ++i) @@ -514,10 +515,19 @@ ExecParallelRetrieveInstrumentation(PlanState *planstate, for (n = 0; n < instrumentation->num_workers; ++n) InstrAggNode(planstate->instrument, &instrument[n]); - /* Also store the per-worker detail. */ + /* + * Also store the per-worker detail. + * + * Worker instrumentation should be allocated in the same context as + * the regular instrumentation information, which is the per-query + * context. Switch into per-query memory context. + */ + oldcontext = MemoryContextSwitchTo(planstate->state->es_query_cxt); ibytes = mul_size(instrumentation->num_workers, sizeof(Instrumentation)); planstate->worker_instrument = palloc(ibytes + offsetof(WorkerInstrumentation, instrument)); + MemoryContextSwitchTo(oldcontext); + planstate->worker_instrument->num_workers = instrumentation->num_workers; memcpy(&planstate->worker_instrument->instrument, instrument, ibytes); From a3bce17ef1ca6408e8d1e7b10fb767aef1729be6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 16 Aug 2016 13:58:44 -0400 Subject: [PATCH 010/871] Automate the maintenance of SO_MINOR_VERSION for our shared libraries. Up to now we've manually adjusted these numbers in several different Makefiles at the start of each development cycle. While that's not much work, it's easily forgotten, so let's get rid of it by setting the SO_MINOR_VERSION values directly from $(MAJORVERSION). In the case of libpq, this dev cycle's value of SO_MINOR_VERSION happens to be "10" anyway, so this switch is transparent. For ecpg's shared libraries, this will result in skipping one or two minor version numbers between v9.6 and v10, which seems like no big problem; and it was a bit inconsistent that they didn't have equal minor version numbers anyway. Discussion: <21969.1471287988@sss.pgh.pa.us> --- src/interfaces/ecpg/compatlib/Makefile | 2 +- src/interfaces/ecpg/ecpglib/Makefile | 2 +- src/interfaces/ecpg/pgtypeslib/Makefile | 2 +- src/interfaces/libpq/Makefile | 2 +- src/tools/RELEASE_CHANGES | 30 +++++++++++++------------ 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/interfaces/ecpg/compatlib/Makefile b/src/interfaces/ecpg/compatlib/Makefile index 2f58ad8592..0b19d832b0 100644 --- a/src/interfaces/ecpg/compatlib/Makefile +++ b/src/interfaces/ecpg/compatlib/Makefile @@ -16,7 +16,7 @@ include $(top_builddir)/src/Makefile.global PGFILEDESC = "ECPG compat - compatibility library for ECPG" NAME= ecpg_compat SO_MAJOR_VERSION= 3 -SO_MINOR_VERSION= 9 +SO_MINOR_VERSION= $(MAJORVERSION) override CPPFLAGS := -I../include -I$(top_srcdir)/src/interfaces/ecpg/include \ -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS) diff --git a/src/interfaces/ecpg/ecpglib/Makefile b/src/interfaces/ecpg/ecpglib/Makefile index 00503b33dd..c9c2499a5e 100644 --- a/src/interfaces/ecpg/ecpglib/Makefile +++ b/src/interfaces/ecpg/ecpglib/Makefile @@ -16,7 +16,7 @@ include $(top_builddir)/src/Makefile.global PGFILEDESC = "ECPG - embedded SQL in C" NAME= ecpg SO_MAJOR_VERSION= 6 -SO_MINOR_VERSION= 9 +SO_MINOR_VERSION= $(MAJORVERSION) override CPPFLAGS := -I../include -I$(top_srcdir)/src/interfaces/ecpg/include \ -I$(libpq_srcdir) -I$(top_builddir)/src/port -DFRONTEND $(CPPFLAGS) diff --git a/src/interfaces/ecpg/pgtypeslib/Makefile b/src/interfaces/ecpg/pgtypeslib/Makefile index ac278948a5..9c9ff08ae8 100644 --- a/src/interfaces/ecpg/pgtypeslib/Makefile +++ b/src/interfaces/ecpg/pgtypeslib/Makefile @@ -16,7 +16,7 @@ include $(top_builddir)/src/Makefile.global PGFILEDESC = "pgtypes - library for data type mapping" NAME= pgtypes SO_MAJOR_VERSION= 3 -SO_MINOR_VERSION= 8 +SO_MINOR_VERSION= $(MAJORVERSION) override CPPFLAGS := -I../include -I$(top_srcdir)/src/interfaces/ecpg/include \ -DFRONTEND $(CPPFLAGS) diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 83b30b0f9e..0b4065ed8f 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -17,7 +17,7 @@ include $(top_builddir)/src/Makefile.global # shared library parameters NAME= pq SO_MAJOR_VERSION= 5 -SO_MINOR_VERSION= 10 +SO_MINOR_VERSION= $(MAJORVERSION) override CPPFLAGS := -DFRONTEND -DUNSAFE_STAT_OK -I$(srcdir) $(CPPFLAGS) -I$(top_builddir)/src/port -I$(top_srcdir)/src/port ifneq ($(PORTNAME), win32) diff --git a/src/tools/RELEASE_CHANGES b/src/tools/RELEASE_CHANGES index e6e294b07e..4f481de12c 100644 --- a/src/tools/RELEASE_CHANGES +++ b/src/tools/RELEASE_CHANGES @@ -73,19 +73,12 @@ Starting a New Development Cycle for example, git push origin master:refs/heads/REL9_2_STABLE +* Add new branch's name to list in src/tools/git_changelog + * Increment the major version number in src/tools/version_stamp.pl * Run "src/tools/version_stamp.pl devel", then run autoconf -* Add version tag to src/tools/git_changelog - -* Bump minor library versions, major if appropriate (see below) - o Look for SO_MINOR_VERSION macros in - src/interfaces/ecpg/compatlib/Makefile - src/interfaces/ecpg/ecpglib/Makefile - src/interfaces/ecpg/pgtypeslib/Makefile - src/interfaces/libpq/Makefile - Creating Back-Branch Release Notes ================================== @@ -139,8 +132,7 @@ function which would give the new field a suitable default value. Adding a new function should NOT force an increase in the major version number. (Packagers will see the standard minor number update and install the new library.) When the major version is increased all applications -which link to the library MUST be recompiled - this is not desirable. When -the major version is updated the minor version gets reset. +which link to the library MUST be recompiled - this is not desirable. Minor Version ============= @@ -150,9 +142,19 @@ the library has changed, typically a change in source code between releases would mean an increase in the minor version number so long as it does not require a major version increase. -Given that we make at least minor changes to our libraries in every major -PostgreSQL version, we always bump all minor library version numbers at the -start of each development cycle as a matter of policy. +Given that we make at least some changes to our libraries in every major +PostgreSQL version, we always bump all minor library version numbers in +each development cycle as a matter of policy. This is currently mechanized +by referencing the MAJORVERSION make macro in the value of SO_MINOR_VERSION +for each shared library. As of v10, SO_MINOR_VERSION is simply equal to +MAJORVERSION in all cases. If we ever make an incompatible break in a +library's API, forcing a major version bump, we could continue to increase +SO_MINOR_VERSION (thus, perhaps, going from libpq.so.5.12 to libpq.so.6.13), +or we could reset SO_MINOR_VERSION to zero, using makefile code along the +lines of + SO_MINOR_VERSION= $(shell expr $(MAJORVERSION) - 13) +so that the number continues to increase automatically in later branches. +For now, that complication is not necessary. Minimizing Changes ================== From f0fe1c8f70bacb65513f1cbaea14eb384d346ee8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 16 Aug 2016 12:00:00 -0400 Subject: [PATCH 011/871] Fix typos From: Alexander Law --- doc/src/sgml/release-9.6.sgml | 2 +- doc/src/sgml/runtime.sgml | 2 +- src/backend/access/transam/multixact.c | 2 +- src/backend/utils/adt/tsquery.c | 2 +- src/test/regress/expected/privileges.out | 2 +- src/test/regress/sql/privileges.sql | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index cc886fa2bb..9003b1f6e4 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -1026,7 +1026,7 @@ This commit is also listed under libpq and psql This view exposes the same information available from - the the pg_config comand-line utility, + the pg_config comand-line utility, namely assorted compile-time configuration information for PostgreSQL. diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 66fbe441ac..60a06590fe 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -184,7 +184,7 @@ postgres$ initdb -D /usr/local/pgsql/data - Non-C and and non-POSIX locales rely on the + Non-C and non-POSIX locales rely on the operating system's collation library for character set ordering. This controls the ordering of keys stored in indexes. For this reason, a cluster cannot switch to an incompatible collation library version, diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index c2e4fa377d..0c8c17af33 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -2802,7 +2802,7 @@ ReadMultiXactCounts(uint32 *multixacts, MultiXactOffset *members) * more aggressive in clamping this value. That not only causes autovacuum * to ramp up, but also makes any manual vacuums the user issues more * aggressive. This happens because vacuum_set_xid_limits() clamps the - * freeze table and and the minimum freeze age based on the effective + * freeze table and the minimum freeze age based on the effective * autovacuum_multixact_freeze_max_age this function returns. In the worst * case, we'll claim the freeze_max_age to zero, and every vacuum of any * table will try to freeze every multixact. diff --git a/src/backend/utils/adt/tsquery.c b/src/backend/utils/adt/tsquery.c index c0a4a0606b..3d11a1c208 100644 --- a/src/backend/utils/adt/tsquery.c +++ b/src/backend/utils/adt/tsquery.c @@ -691,7 +691,7 @@ parse_tsquery(char *buf, findoprnd(ptr, query->size, &needcleanup); /* - * QI_VALSTOP nodes should be cleaned and and OP_PHRASE should be pushed + * QI_VALSTOP nodes should be cleaned and OP_PHRASE should be pushed * down */ if (needcleanup) diff --git a/src/test/regress/expected/privileges.out b/src/test/regress/expected/privileges.out index 996ebcdca2..f66b4432a1 100644 --- a/src/test/regress/expected/privileges.out +++ b/src/test/regress/expected/privileges.out @@ -390,7 +390,7 @@ INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set one = 8; -- f ERROR: permission denied for relation atest5 INSERT INTO atest5(three) VALUES (4) ON CONFLICT (two) DO UPDATE set three = 10; -- fails (due to INSERT) ERROR: permission denied for relation atest5 --- Check that the the columns in the inference require select privileges +-- Check that the columns in the inference require select privileges -- Error. No privs on four INSERT INTO atest5(three) VALUES (4) ON CONFLICT (four) DO UPDATE set three = 10; ERROR: permission denied for relation atest5 diff --git a/src/test/regress/sql/privileges.sql b/src/test/regress/sql/privileges.sql index 0aa9c672d5..00dc7bd4ab 100644 --- a/src/test/regress/sql/privileges.sql +++ b/src/test/regress/sql/privileges.sql @@ -259,7 +259,7 @@ INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = EXCLU INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = EXCLUDED.three; INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set one = 8; -- fails (due to UPDATE) INSERT INTO atest5(three) VALUES (4) ON CONFLICT (two) DO UPDATE set three = 10; -- fails (due to INSERT) --- Check that the the columns in the inference require select privileges +-- Check that the columns in the inference require select privileges -- Error. No privs on four INSERT INTO atest5(three) VALUES (4) ON CONFLICT (four) DO UPDATE set three = 10; From 639166641102871e09f9c4aebc71df57566a0a4a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 16 Aug 2016 12:00:00 -0400 Subject: [PATCH 012/871] doc: Remove some confusion from pg_archivecleanup doc From: Jeff Janes --- doc/src/sgml/ref/pgarchivecleanup.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/pgarchivecleanup.sgml b/doc/src/sgml/ref/pgarchivecleanup.sgml index 60a7fc4e6b..abe01bef4f 100644 --- a/doc/src/sgml/ref/pgarchivecleanup.sgml +++ b/doc/src/sgml/ref/pgarchivecleanup.sgml @@ -122,7 +122,7 @@ pg_archivecleanup: removing file "archive/00000001000000370000000E" extension - When using the program as a standalone utility, provide an extension + Provide an extension that will be stripped from all file names before deciding if they should be deleted. This is typically useful for cleaning up archives that have been compressed during storage, and therefore have had an From 7f61fd10ceb715eceece49451f6dfe9058044e15 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 16 Aug 2016 15:58:30 -0400 Subject: [PATCH 013/871] Fix assorted places in psql to print version numbers >= 10 in new style. This is somewhat cosmetic, since as long as you know what you are looking at, "10.0" is a serviceable substitute for "10". But there is a potential for confusion between version numbers with minor numbers and those without --- we don't want people asking "why is psql saying 10.0 when my server is 10.2". Therefore, back-patch as far as practical, which turns out to be 9.3. I could have redone the patch to use fprintf(stderr) in place of psql_error(), but it seems more work than is warranted for branches that will be EOL or nearly so by the time v10 comes out. Although only psql seems to contain any code that needs this, I chose to put the support function into fe_utils, since it seems likely we'll need it in other client programs in future. (In 9.3-9.5, use dumputils.c, the predecessor of fe_utils/string_utils.c.) In HEAD, also fix the backend code that whines about loadable-library version mismatch. I don't see much need to back-patch that. --- src/backend/utils/fmgr/dfmgr.c | 18 +++-- src/bin/psql/command.c | 50 ++++++++----- src/bin/psql/common.c | 8 ++- src/bin/psql/describe.c | 105 ++++++++++++++++++++-------- src/fe_utils/string_utils.c | 38 ++++++++++ src/include/fe_utils/string_utils.h | 3 + 6 files changed, 167 insertions(+), 55 deletions(-) diff --git a/src/backend/utils/fmgr/dfmgr.c b/src/backend/utils/fmgr/dfmgr.c index f41035d33c..6f70813a6d 100644 --- a/src/backend/utils/fmgr/dfmgr.c +++ b/src/backend/utils/fmgr/dfmgr.c @@ -300,14 +300,22 @@ incompatible_module_error(const char *libname, * block might not even have the fields we expect. */ if (magic_data.version != module_magic_data->version) + { + char library_version[32]; + + if (module_magic_data->version >= 1000) + snprintf(library_version, sizeof(library_version), "%d", + module_magic_data->version / 100); + else + snprintf(library_version, sizeof(library_version), "%d.%d", + module_magic_data->version / 100, + module_magic_data->version % 100); ereport(ERROR, (errmsg("incompatible library \"%s\": version mismatch", libname), - errdetail("Server is version %d.%d, library is version %d.%d.", - magic_data.version / 100, - magic_data.version % 100, - module_magic_data->version / 100, - module_magic_data->version % 100))); + errdetail("Server is version %d, library is version %s.", + magic_data.version / 100, library_version))); + } /* * Otherwise, spell out which fields don't agree. diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 9c0af4e848..4aaf657cce 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -635,8 +635,11 @@ exec_command(const char *cmd, if (pset.sversion < 80400) { - psql_error("The server (version %d.%d) does not support editing function source.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support editing function source.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); status = PSQL_CMD_ERROR; } else if (!query_buf) @@ -731,8 +734,11 @@ exec_command(const char *cmd, if (pset.sversion < 70400) { - psql_error("The server (version %d.%d) does not support editing view definitions.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support editing view definitions.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); status = PSQL_CMD_ERROR; } else if (!query_buf) @@ -1362,8 +1368,11 @@ exec_command(const char *cmd, OT_WHOLE_LINE, NULL, true); if (pset.sversion < 80400) { - psql_error("The server (version %d.%d) does not support showing function source.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support showing function source.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); status = PSQL_CMD_ERROR; } else if (!func) @@ -1441,8 +1450,11 @@ exec_command(const char *cmd, OT_WHOLE_LINE, NULL, true); if (pset.sversion < 70400) { - psql_error("The server (version %d.%d) does not support showing view definitions.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support showing view definitions.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); status = PSQL_CMD_ERROR; } else if (!view) @@ -2014,22 +2026,21 @@ connection_warnings(bool in_startup) if (!pset.quiet && !pset.notty) { int client_ver = PG_VERSION_NUM; + char cverbuf[32]; + char sverbuf[32]; if (pset.sversion != client_ver) { const char *server_version; - char server_ver_str[16]; /* Try to get full text form, might include "devel" etc */ server_version = PQparameterStatus(pset.db, "server_version"); + /* Otherwise fall back on pset.sversion */ if (!server_version) { - snprintf(server_ver_str, sizeof(server_ver_str), - "%d.%d.%d", - pset.sversion / 10000, - (pset.sversion / 100) % 100, - pset.sversion % 100); - server_version = server_ver_str; + formatPGVersionNumber(pset.sversion, true, + sverbuf, sizeof(sverbuf)); + server_version = sverbuf; } printf(_("%s (%s, server %s)\n"), @@ -2040,10 +2051,13 @@ connection_warnings(bool in_startup) printf("%s (%s)\n", pset.progname, PG_VERSION); if (pset.sversion / 100 > client_ver / 100) - printf(_("WARNING: %s major version %d.%d, server major version %d.%d.\n" + printf(_("WARNING: %s major version %s, server major version %s.\n" " Some psql features might not work.\n"), - pset.progname, client_ver / 10000, (client_ver / 100) % 100, - pset.sversion / 10000, (pset.sversion / 100) % 100); + pset.progname, + formatPGVersionNumber(client_ver, false, + cverbuf, sizeof(cverbuf)), + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); #ifdef WIN32 checkWin32Codepage(); diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c index 2450b9c3f8..7399950284 100644 --- a/src/bin/psql/common.c +++ b/src/bin/psql/common.c @@ -18,6 +18,7 @@ #include #endif +#include "fe_utils/string_utils.h" #include "portability/instr_time.h" #include "settings.h" @@ -1202,8 +1203,11 @@ SendQuery(const char *query) { if (on_error_rollback_warning == false && pset.sversion < 80000) { - psql_error("The server (version %d.%d) does not support savepoints for ON_ERROR_ROLLBACK.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support savepoints for ON_ERROR_ROLLBACK.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); on_error_rollback_warning = true; } else diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 27be10215b..6275a688c7 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -142,8 +142,11 @@ describeAccessMethods(const char *pattern, bool verbose) if (pset.sversion < 90600) { - psql_error("The server (version %d.%d) does not support access methods.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support access methods.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -205,8 +208,11 @@ describeTablespaces(const char *pattern, bool verbose) if (pset.sversion < 80000) { - psql_error("The server (version %d.%d) does not support tablespaces.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support tablespaces.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -311,8 +317,11 @@ describeFunctions(const char *functypes, const char *pattern, bool verbose, bool if (showWindow && pset.sversion < 80400) { - psql_error("\\df does not take a \"w\" option with server version %d.%d\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("\\df does not take a \"w\" option with server version %s\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -962,8 +971,11 @@ listDefaultACLs(const char *pattern) if (pset.sversion < 90000) { - psql_error("The server (version %d.%d) does not support altering default privileges.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support altering default privileges.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -3548,8 +3560,11 @@ listCollations(const char *pattern, bool verbose, bool showSystem) if (pset.sversion < 90100) { - psql_error("The server (version %d.%d) does not support collations.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support collations.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -3680,8 +3695,11 @@ listTSParsers(const char *pattern, bool verbose) if (pset.sversion < 80300) { - psql_error("The server (version %d.%d) does not support full text search.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support full text search.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -3915,8 +3933,11 @@ listTSDictionaries(const char *pattern, bool verbose) if (pset.sversion < 80300) { - psql_error("The server (version %d.%d) does not support full text search.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support full text search.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -3983,8 +4004,11 @@ listTSTemplates(const char *pattern, bool verbose) if (pset.sversion < 80300) { - psql_error("The server (version %d.%d) does not support full text search.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support full text search.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -4051,8 +4075,11 @@ listTSConfigs(const char *pattern, bool verbose) if (pset.sversion < 80300) { - psql_error("The server (version %d.%d) does not support full text search.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support full text search.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -4249,8 +4276,11 @@ listForeignDataWrappers(const char *pattern, bool verbose) if (pset.sversion < 80400) { - psql_error("The server (version %d.%d) does not support foreign-data wrappers.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support foreign-data wrappers.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -4329,8 +4359,11 @@ listForeignServers(const char *pattern, bool verbose) if (pset.sversion < 80400) { - psql_error("The server (version %d.%d) does not support foreign servers.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support foreign servers.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -4408,8 +4441,11 @@ listUserMappings(const char *pattern, bool verbose) if (pset.sversion < 80400) { - psql_error("The server (version %d.%d) does not support user mappings.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support user mappings.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -4466,8 +4502,11 @@ listForeignTables(const char *pattern, bool verbose) if (pset.sversion < 90100) { - psql_error("The server (version %d.%d) does not support foreign tables.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support foreign tables.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -4541,8 +4580,11 @@ listExtensions(const char *pattern) if (pset.sversion < 90100) { - psql_error("The server (version %d.%d) does not support extensions.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support extensions.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } @@ -4595,8 +4637,11 @@ listExtensionContents(const char *pattern) if (pset.sversion < 90100) { - psql_error("The server (version %d.%d) does not support extensions.\n", - pset.sversion / 10000, (pset.sversion / 100) % 100); + char sverbuf[32]; + + psql_error("The server (version %s) does not support extensions.\n", + formatPGVersionNumber(pset.sversion, false, + sverbuf, sizeof(sverbuf))); return true; } diff --git a/src/fe_utils/string_utils.c b/src/fe_utils/string_utils.c index f986dbcf39..2c566b1ad7 100644 --- a/src/fe_utils/string_utils.c +++ b/src/fe_utils/string_utils.c @@ -168,6 +168,44 @@ fmtQualifiedId(int remoteVersion, const char *schema, const char *id) } +/* + * Format a Postgres version number (in the PG_VERSION_NUM integer format + * returned by PQserverVersion()) as a string. This exists mainly to + * encapsulate knowledge about two-part vs. three-part version numbers. + * + * For re-entrancy, caller must supply the buffer the string is put in. + * Recommended size of the buffer is 32 bytes. + * + * Returns address of 'buf', as a notational convenience. + */ +char * +formatPGVersionNumber(int version_number, bool include_minor, + char *buf, size_t buflen) +{ + if (version_number >= 100000) + { + /* New two-part style */ + if (include_minor) + snprintf(buf, buflen, "%d.%d", version_number / 10000, + version_number % 10000); + else + snprintf(buf, buflen, "%d", version_number / 10000); + } + else + { + /* Old three-part style */ + if (include_minor) + snprintf(buf, buflen, "%d.%d.%d", version_number / 10000, + (version_number / 100) % 100, + version_number % 100); + else + snprintf(buf, buflen, "%d.%d", version_number / 10000, + (version_number / 100) % 100); + } + return buf; +} + + /* * Convert a string value to an SQL string literal and append it to * the given buffer. We assume the specified client_encoding and diff --git a/src/include/fe_utils/string_utils.h b/src/include/fe_utils/string_utils.h index 7bbed360a3..452ffc0771 100644 --- a/src/include/fe_utils/string_utils.h +++ b/src/include/fe_utils/string_utils.h @@ -30,6 +30,9 @@ extern const char *fmtId(const char *identifier); extern const char *fmtQualifiedId(int remoteVersion, const char *schema, const char *id); +extern char *formatPGVersionNumber(int version_number, bool include_minor, + char *buf, size_t buflen); + extern void appendStringLiteral(PQExpBuffer buf, const char *str, int encoding, bool std_strings); extern void appendStringLiteralConn(PQExpBuffer buf, const char *str, From 4bc4cfe3bd186b4a1d1b01279bfd0e6ab11268b2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 16 Aug 2016 16:14:16 -0400 Subject: [PATCH 014/871] Suppress -Wunused-result warning for strtol(). I'm not sure which bozo thought it's a problem to use strtol() only for its endptr result, but silence the warning using same method used elsewhere. Report: --- src/backend/utils/adt/dbsize.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/adt/dbsize.c b/src/backend/utils/adt/dbsize.c index 0e8a82d6f4..3167bad92b 100644 --- a/src/backend/utils/adt/dbsize.c +++ b/src/backend/utils/adt/dbsize.c @@ -759,13 +759,15 @@ pg_size_bytes(PG_FUNCTION_ARGS) /* Part (4): optional exponent */ if (*endptr == 'e' || *endptr == 'E') { + long exponent; char *cp; /* * Note we might one day support EB units, so if what follows 'E' * isn't a number, just treat it all as a unit to be parsed. */ - (void) strtol(endptr + 1, &cp, 10); + exponent = strtol(endptr + 1, &cp, 10); + (void) exponent; /* Silence -Wunused-result warnings */ if (cp > endptr + 1) endptr = cp; } From 0bb51aa96783e8a6c473c2b5e3725e23e95db834 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 16 Aug 2016 20:33:01 -0400 Subject: [PATCH 015/871] Improve parsetree representation of special functions such as CURRENT_DATE. We implement a dozen or so parameterless functions that the SQL standard defines special syntax for. Up to now, that was done by converting them into more or less ad-hoc constructs such as "'now'::text::date". That's messy for multiple reasons: it exposes what should be implementation details to users, and performance is worse than it needs to be in several cases. To improve matters, invent a new expression node type SQLValueFunction that can represent any of these parameterless functions. Bump catversion because this changes stored parsetrees for rules. Discussion: <30058.1463091294@sss.pgh.pa.us> --- .../pg_stat_statements/pg_stat_statements.c | 9 ++ src/backend/executor/execQual.c | 78 +++++++++++ src/backend/nodes/copyfuncs.c | 19 +++ src/backend/nodes/equalfuncs.c | 14 ++ src/backend/nodes/nodeFuncs.c | 27 +++- src/backend/nodes/outfuncs.c | 14 ++ src/backend/nodes/readfuncs.c | 18 +++ src/backend/optimizer/util/clauses.c | 16 ++- src/backend/parser/gram.y | 124 ++++-------------- src/backend/parser/parse_expr.c | 62 +++++++++ src/backend/parser/parse_target.c | 43 ++++++ src/backend/utils/adt/date.c | 93 ++++++++++++- src/backend/utils/adt/ruleutils.c | 62 +++++++++ src/backend/utils/adt/timestamp.c | 60 +++++++-- src/include/catalog/catversion.h | 2 +- src/include/nodes/nodes.h | 1 + src/include/nodes/primnodes.h | 39 ++++++ src/include/utils/date.h | 4 + src/include/utils/timestamp.h | 4 + src/pl/plpgsql/src/pl_exec.c | 3 + src/test/regress/expected/rowsecurity.out | 16 +-- src/test/regress/expected/rules.out | 6 +- src/test/regress/expected/select_views.out | 40 +++--- src/test/regress/expected/select_views_1.out | 40 +++--- 24 files changed, 626 insertions(+), 168 deletions(-) diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 3d9b8e45d9..8ce24e0401 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -2632,6 +2632,15 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) JumbleExpr(jstate, (Node *) mmexpr->args); } break; + case T_SQLValueFunction: + { + SQLValueFunction *svf = (SQLValueFunction *) node; + + APP_JUMB(svf->op); + /* type is fully determined by op */ + APP_JUMB(svf->typmod); + } + break; case T_XmlExpr: { XmlExpr *xexpr = (XmlExpr *) node; diff --git a/src/backend/executor/execQual.c b/src/backend/executor/execQual.c index cbb76d1f1c..743e7d636a 100644 --- a/src/backend/executor/execQual.c +++ b/src/backend/executor/execQual.c @@ -53,8 +53,10 @@ #include "pgstat.h" #include "utils/acl.h" #include "utils/builtins.h" +#include "utils/date.h" #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "utils/timestamp.h" #include "utils/typcache.h" #include "utils/xml.h" @@ -147,6 +149,9 @@ static Datum ExecEvalCoalesce(CoalesceExprState *coalesceExpr, static Datum ExecEvalMinMax(MinMaxExprState *minmaxExpr, ExprContext *econtext, bool *isNull, ExprDoneCond *isDone); +static Datum ExecEvalSQLValueFunction(ExprState *svfExpr, + ExprContext *econtext, + bool *isNull, ExprDoneCond *isDone); static Datum ExecEvalXml(XmlExprState *xmlExpr, ExprContext *econtext, bool *isNull, ExprDoneCond *isDone); static Datum ExecEvalNullIf(FuncExprState *nullIfExpr, @@ -3530,6 +3535,75 @@ ExecEvalMinMax(MinMaxExprState *minmaxExpr, ExprContext *econtext, return result; } +/* ---------------------------------------------------------------- + * ExecEvalSQLValueFunction + * ---------------------------------------------------------------- + */ +static Datum +ExecEvalSQLValueFunction(ExprState *svfExpr, + ExprContext *econtext, + bool *isNull, ExprDoneCond *isDone) +{ + Datum result = (Datum) 0; + SQLValueFunction *svf = (SQLValueFunction *) svfExpr->expr; + FunctionCallInfoData fcinfo; + + if (isDone) + *isDone = ExprSingleResult; + *isNull = false; + + /* + * Note: current_schema() can return NULL. current_user() etc currently + * cannot, but might as well code those cases the same way for safety. + */ + switch (svf->op) + { + case SVFOP_CURRENT_DATE: + result = DateADTGetDatum(GetSQLCurrentDate()); + break; + case SVFOP_CURRENT_TIME: + case SVFOP_CURRENT_TIME_N: + result = TimeTzADTPGetDatum(GetSQLCurrentTime(svf->typmod)); + break; + case SVFOP_CURRENT_TIMESTAMP: + case SVFOP_CURRENT_TIMESTAMP_N: + result = TimestampTzGetDatum(GetSQLCurrentTimestamp(svf->typmod)); + break; + case SVFOP_LOCALTIME: + case SVFOP_LOCALTIME_N: + result = TimeADTGetDatum(GetSQLLocalTime(svf->typmod)); + break; + case SVFOP_LOCALTIMESTAMP: + case SVFOP_LOCALTIMESTAMP_N: + result = TimestampGetDatum(GetSQLLocalTimestamp(svf->typmod)); + break; + case SVFOP_CURRENT_ROLE: + case SVFOP_CURRENT_USER: + case SVFOP_USER: + InitFunctionCallInfoData(fcinfo, NULL, 0, InvalidOid, NULL, NULL); + result = current_user(&fcinfo); + *isNull = fcinfo.isnull; + break; + case SVFOP_SESSION_USER: + InitFunctionCallInfoData(fcinfo, NULL, 0, InvalidOid, NULL, NULL); + result = session_user(&fcinfo); + *isNull = fcinfo.isnull; + break; + case SVFOP_CURRENT_CATALOG: + InitFunctionCallInfoData(fcinfo, NULL, 0, InvalidOid, NULL, NULL); + result = current_database(&fcinfo); + *isNull = fcinfo.isnull; + break; + case SVFOP_CURRENT_SCHEMA: + InitFunctionCallInfoData(fcinfo, NULL, 0, InvalidOid, NULL, NULL); + result = current_schema(&fcinfo); + *isNull = fcinfo.isnull; + break; + } + + return result; +} + /* ---------------------------------------------------------------- * ExecEvalXml * ---------------------------------------------------------------- @@ -5086,6 +5160,10 @@ ExecInitExpr(Expr *node, PlanState *parent) state = (ExprState *) mstate; } break; + case T_SQLValueFunction: + state = (ExprState *) makeNode(ExprState); + state->evalfunc = ExecEvalSQLValueFunction; + break; case T_XmlExpr: { XmlExpr *xexpr = (XmlExpr *) node; diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 3244c76ddc..c7a06442ba 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -1752,6 +1752,22 @@ _copyMinMaxExpr(const MinMaxExpr *from) return newnode; } +/* + * _copySQLValueFunction + */ +static SQLValueFunction * +_copySQLValueFunction(const SQLValueFunction *from) +{ + SQLValueFunction *newnode = makeNode(SQLValueFunction); + + COPY_SCALAR_FIELD(op); + COPY_SCALAR_FIELD(type); + COPY_SCALAR_FIELD(typmod); + COPY_LOCATION_FIELD(location); + + return newnode; +} + /* * _copyXmlExpr */ @@ -4525,6 +4541,9 @@ copyObject(const void *from) case T_MinMaxExpr: retval = _copyMinMaxExpr(from); break; + case T_SQLValueFunction: + retval = _copySQLValueFunction(from); + break; case T_XmlExpr: retval = _copyXmlExpr(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 1eb679926a..448e1a9d55 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -619,6 +619,17 @@ _equalMinMaxExpr(const MinMaxExpr *a, const MinMaxExpr *b) return true; } +static bool +_equalSQLValueFunction(const SQLValueFunction *a, const SQLValueFunction *b) +{ + COMPARE_SCALAR_FIELD(op); + COMPARE_SCALAR_FIELD(type); + COMPARE_SCALAR_FIELD(typmod); + COMPARE_LOCATION_FIELD(location); + + return true; +} + static bool _equalXmlExpr(const XmlExpr *a, const XmlExpr *b) { @@ -2842,6 +2853,9 @@ equal(const void *a, const void *b) case T_MinMaxExpr: retval = _equalMinMaxExpr(a, b); break; + case T_SQLValueFunction: + retval = _equalSQLValueFunction(a, b); + break; case T_XmlExpr: retval = _equalXmlExpr(a, b); break; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index cd39167351..399744193c 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -218,6 +218,9 @@ exprType(const Node *expr) case T_MinMaxExpr: type = ((const MinMaxExpr *) expr)->minmaxtype; break; + case T_SQLValueFunction: + type = ((const SQLValueFunction *) expr)->type; + break; case T_XmlExpr: if (((const XmlExpr *) expr)->op == IS_DOCUMENT) type = BOOLOID; @@ -479,6 +482,8 @@ exprTypmod(const Node *expr) return typmod; } break; + case T_SQLValueFunction: + return ((const SQLValueFunction *) expr)->typmod; case T_CoerceToDomain: return ((const CoerceToDomain *) expr)->resulttypmod; case T_CoerceToDomainValue: @@ -718,6 +723,8 @@ expression_returns_set_walker(Node *node, void *context) return false; if (IsA(node, MinMaxExpr)) return false; + if (IsA(node, SQLValueFunction)) + return false; if (IsA(node, XmlExpr)) return false; @@ -883,6 +890,9 @@ exprCollation(const Node *expr) case T_MinMaxExpr: coll = ((const MinMaxExpr *) expr)->minmaxcollid; break; + case T_SQLValueFunction: + coll = InvalidOid; /* all cases return non-collatable types */ + break; case T_XmlExpr: /* @@ -1091,6 +1101,9 @@ exprSetCollation(Node *expr, Oid collation) case T_MinMaxExpr: ((MinMaxExpr *) expr)->minmaxcollid = collation; break; + case T_SQLValueFunction: + Assert(!OidIsValid(collation)); /* no collatable results */ + break; case T_XmlExpr: Assert((((XmlExpr *) expr)->op == IS_XMLSERIALIZE) ? (collation == DEFAULT_COLLATION_OID) : @@ -1364,6 +1377,10 @@ exprLocation(const Node *expr) /* GREATEST/LEAST keyword should always be the first thing */ loc = ((const MinMaxExpr *) expr)->location; break; + case T_SQLValueFunction: + /* function keyword should always be the first thing */ + loc = ((const SQLValueFunction *) expr)->location; + break; case T_XmlExpr: { const XmlExpr *xexpr = (const XmlExpr *) expr; @@ -1633,9 +1650,10 @@ set_sa_opfuncid(ScalarArrayOpExpr *opexpr) * for themselves, in case additional checks should be made, or because they * have special rules about which parts of the tree need to be visited. * - * Note: we ignore MinMaxExpr, XmlExpr, and CoerceToDomain nodes, because they - * do not contain SQL function OIDs. However, they can invoke SQL-visible - * functions, so callers should take thought about how to treat them. + * Note: we ignore MinMaxExpr, SQLValueFunction, XmlExpr, and CoerceToDomain + * nodes, because they do not contain SQL function OIDs. However, they can + * invoke SQL-visible functions, so callers should take thought about how to + * treat them. */ bool check_functions_in_node(Node *node, check_function_callback checker, @@ -1859,6 +1877,7 @@ expression_tree_walker(Node *node, case T_CaseTestExpr: case T_SetToDefault: case T_CurrentOfExpr: + case T_SQLValueFunction: case T_RangeTblRef: case T_SortGroupClause: /* primitive node types with no expression subnodes */ @@ -2433,6 +2452,7 @@ expression_tree_mutator(Node *node, case T_CaseTestExpr: case T_SetToDefault: case T_CurrentOfExpr: + case T_SQLValueFunction: case T_RangeTblRef: case T_SortGroupClause: return (Node *) copyObject(node); @@ -3197,6 +3217,7 @@ raw_expression_tree_walker(Node *node, { case T_SetToDefault: case T_CurrentOfExpr: + case T_SQLValueFunction: case T_Integer: case T_Float: case T_String: diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index acaf4ea5eb..1fab807772 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1423,6 +1423,17 @@ _outMinMaxExpr(StringInfo str, const MinMaxExpr *node) WRITE_LOCATION_FIELD(location); } +static void +_outSQLValueFunction(StringInfo str, const SQLValueFunction *node) +{ + WRITE_NODE_TYPE("SQLVALUEFUNCTION"); + + WRITE_ENUM_FIELD(op, SQLValueFunctionOp); + WRITE_OID_FIELD(type); + WRITE_INT_FIELD(typmod); + WRITE_LOCATION_FIELD(location); +} + static void _outXmlExpr(StringInfo str, const XmlExpr *node) { @@ -3522,6 +3533,9 @@ outNode(StringInfo str, const void *obj) case T_MinMaxExpr: _outMinMaxExpr(str, obj); break; + case T_SQLValueFunction: + _outSQLValueFunction(str, obj); + break; case T_XmlExpr: _outXmlExpr(str, obj); break; diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 94954dcc72..c83063e219 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1041,6 +1041,22 @@ _readMinMaxExpr(void) READ_DONE(); } +/* + * _readSQLValueFunction + */ +static SQLValueFunction * +_readSQLValueFunction(void) +{ + READ_LOCALS(SQLValueFunction); + + READ_ENUM_FIELD(op, SQLValueFunctionOp); + READ_OID_FIELD(type); + READ_INT_FIELD(typmod); + READ_LOCATION_FIELD(location); + + READ_DONE(); +} + /* * _readXmlExpr */ @@ -2348,6 +2364,8 @@ parseNodeString(void) return_value = _readCoalesceExpr(); else if (MATCH("MINMAX", 6)) return_value = _readMinMaxExpr(); + else if (MATCH("SQLVALUEFUNCTION", 16)) + return_value = _readSQLValueFunction(); else if (MATCH("XMLEXPR", 7)) return_value = _readXmlExpr(); else if (MATCH("NULLTEST", 8)) diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 4e23898ff9..a40ad40606 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -962,6 +962,12 @@ contain_mutable_functions_walker(Node *node, void *context) context)) return true; + if (IsA(node, SQLValueFunction)) + { + /* all variants of SQLValueFunction are stable */ + return true; + } + /* * It should be safe to treat MinMaxExpr as immutable, because it will * depend on a non-cross-type btree comparison function, and those should @@ -1031,7 +1037,8 @@ contain_volatile_functions_walker(Node *node, void *context) /* * See notes in contain_mutable_functions_walker about why we treat - * MinMaxExpr, XmlExpr, and CoerceToDomain as immutable. + * MinMaxExpr, XmlExpr, and CoerceToDomain as immutable, while + * SQLValueFunction is stable. Hence, none of them are of interest here. */ /* Recurse to check arguments */ @@ -1076,7 +1083,8 @@ contain_volatile_functions_not_nextval_walker(Node *node, void *context) /* * See notes in contain_mutable_functions_walker about why we treat - * MinMaxExpr, XmlExpr, and CoerceToDomain as immutable. + * MinMaxExpr, XmlExpr, and CoerceToDomain as immutable, while + * SQLValueFunction is stable. Hence, none of them are of interest here. */ /* Recurse to check arguments */ @@ -1143,7 +1151,8 @@ has_parallel_hazard_walker(Node *node, has_parallel_hazard_arg *context) * (Note: in principle that's wrong because a domain constraint could * contain a parallel-unsafe function; but useful constraints probably * never would have such, and assuming they do would cripple use of - * parallel query in the presence of domain types.) + * parallel query in the presence of domain types.) SQLValueFunction + * should be safe in all cases. */ if (IsA(node, CoerceToDomain)) { @@ -1458,6 +1467,7 @@ contain_leaked_vars_walker(Node *node, void *context) case T_CaseTestExpr: case T_RowExpr: case T_MinMaxExpr: + case T_SQLValueFunction: case T_NullTest: case T_BooleanTest: case T_List: diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 6a0f7b393c..cb5cfc480c 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -165,6 +165,8 @@ static Node *makeAndExpr(Node *lexpr, Node *rexpr, int location); static Node *makeOrExpr(Node *lexpr, Node *rexpr, int location); static Node *makeNotExpr(Node *expr, int location); static Node *makeAArrayExpr(List *elements, int location); +static Node *makeSQLValueFunction(SQLValueFunctionOp op, int32 typmod, + int location); static Node *makeXmlExpr(XmlExprOp op, char *name, List *named_args, List *args, int location); static List *mergeTableFuncParameters(List *func_args, List *columns); @@ -12330,143 +12332,63 @@ func_expr_common_subexpr: } | CURRENT_DATE { - /* - * Translate as "'now'::text::date". - * - * We cannot use "'now'::date" because coerce_type() will - * immediately reduce that to a constant representing - * today's date. We need to delay the conversion until - * runtime, else the wrong things will happen when - * CURRENT_DATE is used in a column default value or rule. - * - * This could be simplified if we had a way to generate - * an expression tree representing runtime application - * of type-input conversion functions. (As of PG 7.3 - * that is actually possible, but not clear that we want - * to rely on it.) - * - * The token location is attached to the run-time - * typecast, not to the Const, for the convenience of - * pg_stat_statements (which doesn't want these constructs - * to appear to be replaceable constants). - */ - Node *n; - n = makeStringConstCast("now", -1, SystemTypeName("text")); - $$ = makeTypeCast(n, SystemTypeName("date"), @1); + $$ = makeSQLValueFunction(SVFOP_CURRENT_DATE, -1, @1); } | CURRENT_TIME { - /* - * Translate as "'now'::text::timetz". - * See comments for CURRENT_DATE. - */ - Node *n; - n = makeStringConstCast("now", -1, SystemTypeName("text")); - $$ = makeTypeCast(n, SystemTypeName("timetz"), @1); + $$ = makeSQLValueFunction(SVFOP_CURRENT_TIME, -1, @1); } | CURRENT_TIME '(' Iconst ')' { - /* - * Translate as "'now'::text::timetz(n)". - * See comments for CURRENT_DATE. - */ - Node *n; - TypeName *d; - n = makeStringConstCast("now", -1, SystemTypeName("text")); - d = SystemTypeName("timetz"); - d->typmods = list_make1(makeIntConst($3, @3)); - $$ = makeTypeCast(n, d, @1); + $$ = makeSQLValueFunction(SVFOP_CURRENT_TIME_N, $3, @1); } | CURRENT_TIMESTAMP { - /* - * Translate as "now()", since we have a function that - * does exactly what is needed. - */ - $$ = (Node *) makeFuncCall(SystemFuncName("now"), NIL, @1); + $$ = makeSQLValueFunction(SVFOP_CURRENT_TIMESTAMP, -1, @1); } | CURRENT_TIMESTAMP '(' Iconst ')' { - /* - * Translate as "'now'::text::timestamptz(n)". - * See comments for CURRENT_DATE. - */ - Node *n; - TypeName *d; - n = makeStringConstCast("now", -1, SystemTypeName("text")); - d = SystemTypeName("timestamptz"); - d->typmods = list_make1(makeIntConst($3, @3)); - $$ = makeTypeCast(n, d, @1); + $$ = makeSQLValueFunction(SVFOP_CURRENT_TIMESTAMP_N, $3, @1); } | LOCALTIME { - /* - * Translate as "'now'::text::time". - * See comments for CURRENT_DATE. - */ - Node *n; - n = makeStringConstCast("now", -1, SystemTypeName("text")); - $$ = makeTypeCast((Node *)n, SystemTypeName("time"), @1); + $$ = makeSQLValueFunction(SVFOP_LOCALTIME, -1, @1); } | LOCALTIME '(' Iconst ')' { - /* - * Translate as "'now'::text::time(n)". - * See comments for CURRENT_DATE. - */ - Node *n; - TypeName *d; - n = makeStringConstCast("now", -1, SystemTypeName("text")); - d = SystemTypeName("time"); - d->typmods = list_make1(makeIntConst($3, @3)); - $$ = makeTypeCast((Node *)n, d, @1); + $$ = makeSQLValueFunction(SVFOP_LOCALTIME_N, $3, @1); } | LOCALTIMESTAMP { - /* - * Translate as "'now'::text::timestamp". - * See comments for CURRENT_DATE. - */ - Node *n; - n = makeStringConstCast("now", -1, SystemTypeName("text")); - $$ = makeTypeCast(n, SystemTypeName("timestamp"), @1); + $$ = makeSQLValueFunction(SVFOP_LOCALTIMESTAMP, -1, @1); } | LOCALTIMESTAMP '(' Iconst ')' { - /* - * Translate as "'now'::text::timestamp(n)". - * See comments for CURRENT_DATE. - */ - Node *n; - TypeName *d; - n = makeStringConstCast("now", -1, SystemTypeName("text")); - d = SystemTypeName("timestamp"); - d->typmods = list_make1(makeIntConst($3, @3)); - $$ = makeTypeCast(n, d, @1); + $$ = makeSQLValueFunction(SVFOP_LOCALTIMESTAMP_N, $3, @1); } | CURRENT_ROLE { - $$ = (Node *) makeFuncCall(SystemFuncName("current_user"), NIL, @1); + $$ = makeSQLValueFunction(SVFOP_CURRENT_ROLE, -1, @1); } | CURRENT_USER { - $$ = (Node *) makeFuncCall(SystemFuncName("current_user"), NIL, @1); + $$ = makeSQLValueFunction(SVFOP_CURRENT_USER, -1, @1); } | SESSION_USER { - $$ = (Node *) makeFuncCall(SystemFuncName("session_user"), NIL, @1); + $$ = makeSQLValueFunction(SVFOP_SESSION_USER, -1, @1); } | USER { - $$ = (Node *) makeFuncCall(SystemFuncName("current_user"), NIL, @1); + $$ = makeSQLValueFunction(SVFOP_USER, -1, @1); } | CURRENT_CATALOG { - $$ = (Node *) makeFuncCall(SystemFuncName("current_database"), NIL, @1); + $$ = makeSQLValueFunction(SVFOP_CURRENT_CATALOG, -1, @1); } | CURRENT_SCHEMA { - $$ = (Node *) makeFuncCall(SystemFuncName("current_schema"), NIL, @1); + $$ = makeSQLValueFunction(SVFOP_CURRENT_SCHEMA, -1, @1); } | CAST '(' a_expr AS Typename ')' { $$ = makeTypeCast($3, $5, @1); } @@ -14710,6 +14632,18 @@ makeAArrayExpr(List *elements, int location) return (Node *) n; } +static Node * +makeSQLValueFunction(SQLValueFunctionOp op, int32 typmod, int location) +{ + SQLValueFunction *svf = makeNode(SQLValueFunction); + + svf->op = op; + /* svf->type will be filled during parse analysis */ + svf->typmod = typmod; + svf->location = location; + return (Node *) svf; +} + static Node * makeXmlExpr(XmlExprOp op, char *name, List *named_args, List *args, int location) diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c index cead21283d..63f7965532 100644 --- a/src/backend/parser/parse_expr.c +++ b/src/backend/parser/parse_expr.c @@ -34,7 +34,9 @@ #include "parser/parse_type.h" #include "parser/parse_agg.h" #include "utils/builtins.h" +#include "utils/date.h" #include "utils/lsyscache.h" +#include "utils/timestamp.h" #include "utils/xml.h" @@ -107,6 +109,8 @@ static Node *transformArrayExpr(ParseState *pstate, A_ArrayExpr *a, static Node *transformRowExpr(ParseState *pstate, RowExpr *r); static Node *transformCoalesceExpr(ParseState *pstate, CoalesceExpr *c); static Node *transformMinMaxExpr(ParseState *pstate, MinMaxExpr *m); +static Node *transformSQLValueFunction(ParseState *pstate, + SQLValueFunction *svf); static Node *transformXmlExpr(ParseState *pstate, XmlExpr *x); static Node *transformXmlSerialize(ParseState *pstate, XmlSerialize *xs); static Node *transformBooleanTest(ParseState *pstate, BooleanTest *b); @@ -306,6 +310,11 @@ transformExprRecurse(ParseState *pstate, Node *expr) result = transformMinMaxExpr(pstate, (MinMaxExpr *) expr); break; + case T_SQLValueFunction: + result = transformSQLValueFunction(pstate, + (SQLValueFunction *) expr); + break; + case T_XmlExpr: result = transformXmlExpr(pstate, (XmlExpr *) expr); break; @@ -2178,6 +2187,59 @@ transformMinMaxExpr(ParseState *pstate, MinMaxExpr *m) return (Node *) newm; } +static Node * +transformSQLValueFunction(ParseState *pstate, SQLValueFunction *svf) +{ + /* + * All we need to do is insert the correct result type and (where needed) + * validate the typmod, so we just modify the node in-place. + */ + switch (svf->op) + { + case SVFOP_CURRENT_DATE: + svf->type = DATEOID; + break; + case SVFOP_CURRENT_TIME: + svf->type = TIMETZOID; + break; + case SVFOP_CURRENT_TIME_N: + svf->type = TIMETZOID; + svf->typmod = anytime_typmod_check(true, svf->typmod); + break; + case SVFOP_CURRENT_TIMESTAMP: + svf->type = TIMESTAMPTZOID; + break; + case SVFOP_CURRENT_TIMESTAMP_N: + svf->type = TIMESTAMPTZOID; + svf->typmod = anytimestamp_typmod_check(true, svf->typmod); + break; + case SVFOP_LOCALTIME: + svf->type = TIMEOID; + break; + case SVFOP_LOCALTIME_N: + svf->type = TIMEOID; + svf->typmod = anytime_typmod_check(false, svf->typmod); + break; + case SVFOP_LOCALTIMESTAMP: + svf->type = TIMESTAMPOID; + break; + case SVFOP_LOCALTIMESTAMP_N: + svf->type = TIMESTAMPOID; + svf->typmod = anytimestamp_typmod_check(false, svf->typmod); + break; + case SVFOP_CURRENT_ROLE: + case SVFOP_CURRENT_USER: + case SVFOP_USER: + case SVFOP_SESSION_USER: + case SVFOP_CURRENT_CATALOG: + case SVFOP_CURRENT_SCHEMA: + svf->type = NAMEOID; + break; + } + + return (Node *) svf; +} + static Node * transformXmlExpr(ParseState *pstate, XmlExpr *x) { diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c index fc93063ed0..b7b82bfb6b 100644 --- a/src/backend/parser/parse_target.c +++ b/src/backend/parser/parse_target.c @@ -1761,6 +1761,49 @@ FigureColnameInternal(Node *node, char **name) return 2; } break; + case T_SQLValueFunction: + /* make these act like a function or variable */ + switch (((SQLValueFunction *) node)->op) + { + case SVFOP_CURRENT_DATE: + *name = "current_date"; + return 2; + case SVFOP_CURRENT_TIME: + case SVFOP_CURRENT_TIME_N: + *name = "current_time"; + return 2; + case SVFOP_CURRENT_TIMESTAMP: + case SVFOP_CURRENT_TIMESTAMP_N: + *name = "current_timestamp"; + return 2; + case SVFOP_LOCALTIME: + case SVFOP_LOCALTIME_N: + *name = "localtime"; + return 2; + case SVFOP_LOCALTIMESTAMP: + case SVFOP_LOCALTIMESTAMP_N: + *name = "localtimestamp"; + return 2; + case SVFOP_CURRENT_ROLE: + *name = "current_role"; + return 2; + case SVFOP_CURRENT_USER: + *name = "current_user"; + return 2; + case SVFOP_USER: + *name = "user"; + return 2; + case SVFOP_SESSION_USER: + *name = "session_user"; + return 2; + case SVFOP_CURRENT_CATALOG: + *name = "current_catalog"; + return 2; + case SVFOP_CURRENT_SCHEMA: + *name = "current_schema"; + return 2; + } + break; case T_XmlExpr: /* make SQL/XML functions act like a regular function */ switch (((XmlExpr *) node)->op) diff --git a/src/backend/utils/adt/date.c b/src/backend/utils/adt/date.c index 420f383a80..bc7d190210 100644 --- a/src/backend/utils/adt/date.c +++ b/src/backend/utils/adt/date.c @@ -21,6 +21,7 @@ #include #include "access/hash.h" +#include "access/xact.h" #include "libpq/pqformat.h" #include "miscadmin.h" #include "parser/scansup.h" @@ -51,7 +52,6 @@ static void AdjustTimeForTypmod(TimeADT *time, int32 typmod); static int32 anytime_typmodin(bool istz, ArrayType *ta) { - int32 typmod; int32 *tl; int n; @@ -66,22 +66,27 @@ anytime_typmodin(bool istz, ArrayType *ta) (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid type modifier"))); - if (*tl < 0) + return anytime_typmod_check(istz, tl[0]); +} + +/* exported so parse_expr.c can use it */ +int32 +anytime_typmod_check(bool istz, int32 typmod) +{ + if (typmod < 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("TIME(%d)%s precision must not be negative", - *tl, (istz ? " WITH TIME ZONE" : "")))); - if (*tl > MAX_TIME_PRECISION) + typmod, (istz ? " WITH TIME ZONE" : "")))); + if (typmod > MAX_TIME_PRECISION) { ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("TIME(%d)%s precision reduced to maximum allowed, %d", - *tl, (istz ? " WITH TIME ZONE" : ""), + typmod, (istz ? " WITH TIME ZONE" : ""), MAX_TIME_PRECISION))); typmod = MAX_TIME_PRECISION; } - else - typmod = *tl; return typmod; } @@ -298,6 +303,80 @@ EncodeSpecialDate(DateADT dt, char *str) } +/* + * GetSQLCurrentDate -- implements CURRENT_DATE + */ +DateADT +GetSQLCurrentDate(void) +{ + TimestampTz ts; + struct pg_tm tt, + *tm = &tt; + fsec_t fsec; + int tz; + + ts = GetCurrentTransactionStartTimestamp(); + + if (timestamp2tm(ts, &tz, tm, &fsec, NULL, NULL) != 0) + ereport(ERROR, + (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), + errmsg("timestamp out of range"))); + + return date2j(tm->tm_year, tm->tm_mon, tm->tm_mday) - POSTGRES_EPOCH_JDATE; +} + +/* + * GetSQLCurrentTime -- implements CURRENT_TIME, CURRENT_TIME(n) + */ +TimeTzADT * +GetSQLCurrentTime(int32 typmod) +{ + TimeTzADT *result; + TimestampTz ts; + struct pg_tm tt, + *tm = &tt; + fsec_t fsec; + int tz; + + ts = GetCurrentTransactionStartTimestamp(); + + if (timestamp2tm(ts, &tz, tm, &fsec, NULL, NULL) != 0) + ereport(ERROR, + (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), + errmsg("timestamp out of range"))); + + result = (TimeTzADT *) palloc(sizeof(TimeTzADT)); + tm2timetz(tm, fsec, tz, result); + AdjustTimeForTypmod(&(result->time), typmod); + return result; +} + +/* + * GetSQLLocalTime -- implements LOCALTIME, LOCALTIME(n) + */ +TimeADT +GetSQLLocalTime(int32 typmod) +{ + TimeADT result; + TimestampTz ts; + struct pg_tm tt, + *tm = &tt; + fsec_t fsec; + int tz; + + ts = GetCurrentTransactionStartTimestamp(); + + if (timestamp2tm(ts, &tz, tm, &fsec, NULL, NULL) != 0) + ereport(ERROR, + (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), + errmsg("timestamp out of range"))); + + tm2time(tm, fsec, &result); + AdjustTimeForTypmod(&result, typmod); + return result; +} + + /* * Comparison functions for dates */ diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index ec966c752e..8a81d7a078 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -6884,6 +6884,7 @@ isSimpleNode(Node *node, Node *parentNode, int prettyFlags) case T_RowExpr: case T_CoalesceExpr: case T_MinMaxExpr: + case T_SQLValueFunction: case T_XmlExpr: case T_NullIfExpr: case T_Aggref: @@ -7871,6 +7872,67 @@ get_rule_expr(Node *node, deparse_context *context, } break; + case T_SQLValueFunction: + { + SQLValueFunction *svf = (SQLValueFunction *) node; + + /* + * Note: this code knows that typmod for time, timestamp, and + * timestamptz just prints as integer. + */ + switch (svf->op) + { + case SVFOP_CURRENT_DATE: + appendStringInfoString(buf, "CURRENT_DATE"); + break; + case SVFOP_CURRENT_TIME: + appendStringInfoString(buf, "CURRENT_TIME"); + break; + case SVFOP_CURRENT_TIME_N: + appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); + break; + case SVFOP_CURRENT_TIMESTAMP: + appendStringInfoString(buf, "CURRENT_TIMESTAMP"); + break; + case SVFOP_CURRENT_TIMESTAMP_N: + appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_LOCALTIME: + appendStringInfoString(buf, "LOCALTIME"); + break; + case SVFOP_LOCALTIME_N: + appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); + break; + case SVFOP_LOCALTIMESTAMP: + appendStringInfoString(buf, "LOCALTIMESTAMP"); + break; + case SVFOP_LOCALTIMESTAMP_N: + appendStringInfo(buf, "LOCALTIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_CURRENT_ROLE: + appendStringInfoString(buf, "CURRENT_ROLE"); + break; + case SVFOP_CURRENT_USER: + appendStringInfoString(buf, "CURRENT_USER"); + break; + case SVFOP_USER: + appendStringInfoString(buf, "USER"); + break; + case SVFOP_SESSION_USER: + appendStringInfoString(buf, "SESSION_USER"); + break; + case SVFOP_CURRENT_CATALOG: + appendStringInfoString(buf, "CURRENT_CATALOG"); + break; + case SVFOP_CURRENT_SCHEMA: + appendStringInfoString(buf, "CURRENT_SCHEMA"); + break; + } + } + break; + case T_XmlExpr: { XmlExpr *xexpr = (XmlExpr *) node; diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c index d7ee865cf7..c1d6f05b5e 100644 --- a/src/backend/utils/adt/timestamp.c +++ b/src/backend/utils/adt/timestamp.c @@ -72,13 +72,13 @@ static Timestamp dt2local(Timestamp dt, int timezone); static void AdjustTimestampForTypmod(Timestamp *time, int32 typmod); static void AdjustIntervalForTypmod(Interval *interval, int32 typmod); static TimestampTz timestamp2timestamptz(Timestamp timestamp); +static Timestamp timestamptz2timestamp(TimestampTz timestamp); /* common code for timestamptypmodin and timestamptztypmodin */ static int32 anytimestamp_typmodin(bool istz, ArrayType *ta) { - int32 typmod; int32 *tl; int n; @@ -93,22 +93,27 @@ anytimestamp_typmodin(bool istz, ArrayType *ta) (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid type modifier"))); - if (*tl < 0) + return anytimestamp_typmod_check(istz, tl[0]); +} + +/* exported so parse_expr.c can use it */ +int32 +anytimestamp_typmod_check(bool istz, int32 typmod) +{ + if (typmod < 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("TIMESTAMP(%d)%s precision must not be negative", - *tl, (istz ? " WITH TIME ZONE" : "")))); - if (*tl > MAX_TIMESTAMP_PRECISION) + typmod, (istz ? " WITH TIME ZONE" : "")))); + if (typmod > MAX_TIMESTAMP_PRECISION) { ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("TIMESTAMP(%d)%s precision reduced to maximum allowed, %d", - *tl, (istz ? " WITH TIME ZONE" : ""), + typmod, (istz ? " WITH TIME ZONE" : ""), MAX_TIMESTAMP_PRECISION))); typmod = MAX_TIMESTAMP_PRECISION; } - else - typmod = *tl; return typmod; } @@ -336,6 +341,10 @@ timestamp_scale(PG_FUNCTION_ARGS) PG_RETURN_TIMESTAMP(result); } +/* + * AdjustTimestampForTypmod --- round off a timestamp to suit given typmod + * Works for either timestamp or timestamptz. + */ static void AdjustTimestampForTypmod(Timestamp *time, int32 typmod) { @@ -1686,6 +1695,34 @@ IntegerTimestampToTimestampTz(int64 timestamp) } #endif +/* + * GetSQLCurrentTimestamp -- implements CURRENT_TIMESTAMP, CURRENT_TIMESTAMP(n) + */ +TimestampTz +GetSQLCurrentTimestamp(int32 typmod) +{ + TimestampTz ts; + + ts = GetCurrentTransactionStartTimestamp(); + if (typmod >= 0) + AdjustTimestampForTypmod(&ts, typmod); + return ts; +} + +/* + * GetSQLLocalTimestamp -- implements LOCALTIMESTAMP, LOCALTIMESTAMP(n) + */ +Timestamp +GetSQLLocalTimestamp(int32 typmod) +{ + Timestamp ts; + + ts = timestamptz2timestamp(GetCurrentTransactionStartTimestamp()); + if (typmod >= 0) + AdjustTimestampForTypmod(&ts, typmod); + return ts; +} + /* * TimestampDifference -- convert the difference between two timestamps * into integer seconds and microseconds @@ -5415,6 +5452,13 @@ Datum timestamptz_timestamp(PG_FUNCTION_ARGS) { TimestampTz timestamp = PG_GETARG_TIMESTAMPTZ(0); + + PG_RETURN_TIMESTAMP(timestamptz2timestamp(timestamp)); +} + +static Timestamp +timestamptz2timestamp(TimestampTz timestamp) +{ Timestamp result; struct pg_tm tt, *tm = &tt; @@ -5434,7 +5478,7 @@ timestamptz_timestamp(PG_FUNCTION_ARGS) (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), errmsg("timestamp out of range"))); } - PG_RETURN_TIMESTAMP(result); + return result; } /* timestamptz_zone() diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 2ca3cd911a..82810c8fba 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201608131 +#define CATALOG_VERSION_NO 201608161 #endif diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 6b850e4bc4..2f7efa810c 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -166,6 +166,7 @@ typedef enum NodeTag T_RowCompareExpr, T_CoalesceExpr, T_MinMaxExpr, + T_SQLValueFunction, T_XmlExpr, T_NullTest, T_BooleanTest, diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index df2d27d77c..65510b010b 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -1050,6 +1050,45 @@ typedef struct MinMaxExpr int location; /* token location, or -1 if unknown */ } MinMaxExpr; +/* + * SQLValueFunction - parameterless functions with special grammar productions + * + * The SQL standard categorizes some of these as + * and others as . We call 'em SQLValueFunctions + * for lack of a better term. We store type and typmod of the result so that + * some code doesn't need to know each function individually, and because + * we would need to store typmod anyway for some of the datetime functions. + * Note that currently, all variants return non-collating datatypes, so we do + * not need a collation field; also, all these functions are stable. + */ +typedef enum SQLValueFunctionOp +{ + SVFOP_CURRENT_DATE, + SVFOP_CURRENT_TIME, + SVFOP_CURRENT_TIME_N, + SVFOP_CURRENT_TIMESTAMP, + SVFOP_CURRENT_TIMESTAMP_N, + SVFOP_LOCALTIME, + SVFOP_LOCALTIME_N, + SVFOP_LOCALTIMESTAMP, + SVFOP_LOCALTIMESTAMP_N, + SVFOP_CURRENT_ROLE, + SVFOP_CURRENT_USER, + SVFOP_USER, + SVFOP_SESSION_USER, + SVFOP_CURRENT_CATALOG, + SVFOP_CURRENT_SCHEMA +} SQLValueFunctionOp; + +typedef struct SQLValueFunction +{ + Expr xpr; + SQLValueFunctionOp op; /* which function this is */ + Oid type; /* result type/typmod */ + int32 typmod; + int location; /* token location, or -1 if unknown */ +} SQLValueFunction; + /* * XmlExpr - various SQL/XML functions requiring special grammar productions * diff --git a/src/include/utils/date.h b/src/include/utils/date.h index 1b962af7d8..df753c4450 100644 --- a/src/include/utils/date.h +++ b/src/include/utils/date.h @@ -89,8 +89,12 @@ typedef struct /* date.c */ +extern int32 anytime_typmod_check(bool istz, int32 typmod); extern double date2timestamp_no_overflow(DateADT dateVal); extern void EncodeSpecialDate(DateADT dt, char *str); +extern DateADT GetSQLCurrentDate(void); +extern TimeTzADT *GetSQLCurrentTime(int32 typmod); +extern TimeADT GetSQLLocalTime(int32 typmod); extern Datum date_in(PG_FUNCTION_ARGS); extern Datum date_out(PG_FUNCTION_ARGS); diff --git a/src/include/utils/timestamp.h b/src/include/utils/timestamp.h index 85cc7ce1fe..93b90fe3a0 100644 --- a/src/include/utils/timestamp.h +++ b/src/include/utils/timestamp.h @@ -215,7 +215,11 @@ extern Datum generate_series_timestamptz(PG_FUNCTION_ARGS); /* Internal routines (not fmgr-callable) */ +extern int32 anytimestamp_typmod_check(bool istz, int32 typmod); + extern TimestampTz GetCurrentTimestamp(void); +extern TimestampTz GetSQLCurrentTimestamp(int32 typmod); +extern Timestamp GetSQLLocalTimestamp(int32 typmod); extern void TimestampDifference(TimestampTz start_time, TimestampTz stop_time, long *secs, int *microsecs); extern bool TimestampDifferenceExceeds(TimestampTz start_time, diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index 586ff1f329..fec55e502f 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -6494,6 +6494,9 @@ exec_simple_check_node(Node *node) return TRUE; } + case T_SQLValueFunction: + return TRUE; + case T_XmlExpr: { XmlExpr *expr = (XmlExpr *) node; diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out index c15bf958a5..abfee92f4d 100644 --- a/src/test/regress/expected/rowsecurity.out +++ b/src/test/regress/expected/rowsecurity.out @@ -182,7 +182,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); Filter: (dlevel <= $0) InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount - Index Cond: (pguser = "current_user"()) + Index Cond: (pguser = CURRENT_USER) (7 rows) EXPLAIN (COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); @@ -198,7 +198,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dt Filter: (dlevel <= $0) InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount - Index Cond: (pguser = "current_user"()) + Index Cond: (pguser = CURRENT_USER) (11 rows) -- only owner can change policies @@ -265,22 +265,22 @@ NOTICE: f_leak => great manga (3 rows) EXPLAIN (COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); - QUERY PLAN ----------------------------------------------- + QUERY PLAN +------------------------------------------ Subquery Scan on document Filter: f_leak(document.dtitle) -> Seq Scan on document document_1 - Filter: (dauthor = "current_user"()) + Filter: (dauthor = CURRENT_USER) (4 rows) EXPLAIN (COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); - QUERY PLAN ----------------------------------------------------- + QUERY PLAN +-------------------------------------------------- Nested Loop -> Subquery Scan on document Filter: f_leak(document.dtitle) -> Seq Scan on document document_1 - Filter: (dauthor = "current_user"()) + Filter: (dauthor = CURRENT_USER) -> Index Scan using category_pkey on category Index Cond: (cid = document.cid) (7 rows) diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c5ff3181a3..8157324fee 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2269,14 +2269,14 @@ pg_settings|pg_settings_u|CREATE RULE pg_settings_u AS WHERE (new.name = old.name) DO SELECT set_config(old.name, new.setting, false) AS set_config; rtest_emp|rtest_emp_del|CREATE RULE rtest_emp_del AS ON DELETE TO rtest_emp DO INSERT INTO rtest_emplog (ename, who, action, newsal, oldsal) - VALUES (old.ename, "current_user"(), 'fired'::bpchar, '$0.00'::money, old.salary); + VALUES (old.ename, CURRENT_USER, 'fired'::bpchar, '$0.00'::money, old.salary); rtest_emp|rtest_emp_ins|CREATE RULE rtest_emp_ins AS ON INSERT TO rtest_emp DO INSERT INTO rtest_emplog (ename, who, action, newsal, oldsal) - VALUES (new.ename, "current_user"(), 'hired'::bpchar, new.salary, '$0.00'::money); + VALUES (new.ename, CURRENT_USER, 'hired'::bpchar, new.salary, '$0.00'::money); rtest_emp|rtest_emp_upd|CREATE RULE rtest_emp_upd AS ON UPDATE TO rtest_emp WHERE (new.salary <> old.salary) DO INSERT INTO rtest_emplog (ename, who, action, newsal, oldsal) - VALUES (new.ename, "current_user"(), 'honored'::bpchar, new.salary, old.salary); + VALUES (new.ename, CURRENT_USER, 'honored'::bpchar, new.salary, old.salary); rtest_nothn1|rtest_nothn_r1|CREATE RULE rtest_nothn_r1 AS ON INSERT TO rtest_nothn1 WHERE ((new.a >= 10) AND (new.a < 20)) DO INSTEAD NOTHING; diff --git a/src/test/regress/expected/select_views.out b/src/test/regress/expected/select_views.out index 7f575266c1..878035332b 100644 --- a/src/test/regress/expected/select_views.out +++ b/src/test/regress/expected/select_views.out @@ -1326,10 +1326,10 @@ NOTICE: f_leak => hamburger (1 row) EXPLAIN (COSTS OFF) SELECT * FROM my_property_normal WHERE f_leak(passwd); - QUERY PLAN ------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------- Seq Scan on customer - Filter: (f_leak(passwd) AND (name = ("current_user"())::text)) + Filter: (f_leak(passwd) AND (name = (CURRENT_USER)::text)) (2 rows) SELECT * FROM my_property_secure WHERE f_leak(passwd); @@ -1340,12 +1340,12 @@ NOTICE: f_leak => passwd123 (1 row) EXPLAIN (COSTS OFF) SELECT * FROM my_property_secure WHERE f_leak(passwd); - QUERY PLAN ---------------------------------------------------- + QUERY PLAN +----------------------------------------------- Subquery Scan on my_property_secure Filter: f_leak(my_property_secure.passwd) -> Seq Scan on customer - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (4 rows) -- @@ -1367,10 +1367,10 @@ NOTICE: f_leak => hamburger EXPLAIN (COSTS OFF) SELECT * FROM my_property_normal v WHERE f_leak('passwd') AND f_leak(passwd); - QUERY PLAN ---------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------- Seq Scan on customer - Filter: (f_leak('passwd'::text) AND f_leak(passwd) AND (name = ("current_user"())::text)) + Filter: (f_leak('passwd'::text) AND f_leak(passwd) AND (name = (CURRENT_USER)::text)) (2 rows) SELECT * FROM my_property_secure v @@ -1386,12 +1386,12 @@ NOTICE: f_leak => passwd EXPLAIN (COSTS OFF) SELECT * FROM my_property_secure v WHERE f_leak('passwd') AND f_leak(passwd); - QUERY PLAN --------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------- Subquery Scan on v Filter: f_leak(v.passwd) -> Seq Scan on customer - Filter: (f_leak('passwd'::text) AND (name = ("current_user"())::text)) + Filter: (f_leak('passwd'::text) AND (name = (CURRENT_USER)::text)) (4 rows) -- @@ -1409,15 +1409,15 @@ NOTICE: f_leak => 9801-2345-6789-0123 (1 row) EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_normal WHERE f_leak(cnum); - QUERY PLAN ---------------------------------------------------------- + QUERY PLAN +----------------------------------------------------- Hash Join Hash Cond: (r.cid = l.cid) -> Seq Scan on credit_card r Filter: f_leak(cnum) -> Hash -> Seq Scan on customer l - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (7 rows) SELECT * FROM my_credit_card_secure WHERE f_leak(cnum); @@ -1428,8 +1428,8 @@ NOTICE: f_leak => 1111-2222-3333-4444 (1 row) EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_secure WHERE f_leak(cnum); - QUERY PLAN ---------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------- Subquery Scan on my_credit_card_secure Filter: f_leak(my_credit_card_secure.cnum) -> Hash Join @@ -1437,7 +1437,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_secure WHERE f_leak(cnum); -> Seq Scan on credit_card r -> Hash -> Seq Scan on customer l - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (8 rows) -- @@ -1471,7 +1471,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_usage_normal -> Seq Scan on credit_card r_1 -> Hash -> Seq Scan on customer l_1 - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (13 rows) SELECT * FROM my_credit_card_usage_secure @@ -1502,7 +1502,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_usage_secure -> Seq Scan on credit_card r_1 -> Hash -> Seq Scan on customer l - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (13 rows) -- diff --git a/src/test/regress/expected/select_views_1.out b/src/test/regress/expected/select_views_1.out index 5275ef0b2d..1a05c6ccbd 100644 --- a/src/test/regress/expected/select_views_1.out +++ b/src/test/regress/expected/select_views_1.out @@ -1326,10 +1326,10 @@ NOTICE: f_leak => hamburger (1 row) EXPLAIN (COSTS OFF) SELECT * FROM my_property_normal WHERE f_leak(passwd); - QUERY PLAN ------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------- Seq Scan on customer - Filter: (f_leak(passwd) AND (name = ("current_user"())::text)) + Filter: (f_leak(passwd) AND (name = (CURRENT_USER)::text)) (2 rows) SELECT * FROM my_property_secure WHERE f_leak(passwd); @@ -1340,12 +1340,12 @@ NOTICE: f_leak => passwd123 (1 row) EXPLAIN (COSTS OFF) SELECT * FROM my_property_secure WHERE f_leak(passwd); - QUERY PLAN ---------------------------------------------------- + QUERY PLAN +----------------------------------------------- Subquery Scan on my_property_secure Filter: f_leak(my_property_secure.passwd) -> Seq Scan on customer - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (4 rows) -- @@ -1367,10 +1367,10 @@ NOTICE: f_leak => hamburger EXPLAIN (COSTS OFF) SELECT * FROM my_property_normal v WHERE f_leak('passwd') AND f_leak(passwd); - QUERY PLAN ---------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------- Seq Scan on customer - Filter: (f_leak('passwd'::text) AND f_leak(passwd) AND (name = ("current_user"())::text)) + Filter: (f_leak('passwd'::text) AND f_leak(passwd) AND (name = (CURRENT_USER)::text)) (2 rows) SELECT * FROM my_property_secure v @@ -1386,12 +1386,12 @@ NOTICE: f_leak => passwd EXPLAIN (COSTS OFF) SELECT * FROM my_property_secure v WHERE f_leak('passwd') AND f_leak(passwd); - QUERY PLAN --------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------- Subquery Scan on v Filter: f_leak(v.passwd) -> Seq Scan on customer - Filter: (f_leak('passwd'::text) AND (name = ("current_user"())::text)) + Filter: (f_leak('passwd'::text) AND (name = (CURRENT_USER)::text)) (4 rows) -- @@ -1409,15 +1409,15 @@ NOTICE: f_leak => 9801-2345-6789-0123 (1 row) EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_normal WHERE f_leak(cnum); - QUERY PLAN ---------------------------------------------------------- + QUERY PLAN +----------------------------------------------------- Hash Join Hash Cond: (r.cid = l.cid) -> Seq Scan on credit_card r Filter: f_leak(cnum) -> Hash -> Seq Scan on customer l - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (7 rows) SELECT * FROM my_credit_card_secure WHERE f_leak(cnum); @@ -1428,8 +1428,8 @@ NOTICE: f_leak => 1111-2222-3333-4444 (1 row) EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_secure WHERE f_leak(cnum); - QUERY PLAN ---------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------- Subquery Scan on my_credit_card_secure Filter: f_leak(my_credit_card_secure.cnum) -> Hash Join @@ -1437,7 +1437,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_secure WHERE f_leak(cnum); -> Seq Scan on credit_card r -> Hash -> Seq Scan on customer l - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (8 rows) -- @@ -1471,7 +1471,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_usage_normal -> Seq Scan on credit_card r_1 -> Hash -> Seq Scan on customer l_1 - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (13 rows) SELECT * FROM my_credit_card_usage_secure @@ -1502,7 +1502,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM my_credit_card_usage_secure -> Seq Scan on credit_card r_1 -> Hash -> Seq Scan on customer l - Filter: (name = ("current_user"())::text) + Filter: (name = (CURRENT_USER)::text) (13 rows) -- From d125d25790901683d5ad16bfc96e9de4ccc9a581 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 16 Aug 2016 23:04:50 -0400 Subject: [PATCH 016/871] docs: my third pass over the 9.6 release notes Backpatch-through: 9.6 --- doc/src/sgml/release-9.6.sgml | 628 ++++++++++++++++++---------------- 1 file changed, 332 insertions(+), 296 deletions(-) diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 9003b1f6e4..8d7356e27f 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -401,7 +401,7 @@ This commit is also listed under libpq and psql Allow GIN index builds to make effective use of - settings larger than 1GB (Robert Abraham, Teodor Sigaev) + settings larger than 1 GB (Robert Abraham, Teodor Sigaev) @@ -467,55 +467,130 @@ This commit is also listed under libpq and psql - General Performance + Sorting - Avoid re-vacuuming pages containing only frozen tuples (Masahiko - Sawada, Robert Haas, Andres Freund) + Improve sorting performance by using quicksort, not replacement + selection sort, when performing external sort steps (Peter + Geoghegan) - Formerly, anti-wraparound vacuum had to visit every page of - a table, even pages where there was nothing to do. Now, pages - containing only already-frozen tuples are identified in the table's - visibility map, and can be skipped by vacuum even when doing - transaction wraparound prevention. This should greatly reduce the - cost of maintaining large tables containing mostly-unchanged data. + The new approach makes better use of the CPU cache + for typical cache sizes and data volumes. Where necessary, + the behavior can be adjusted via the new configuration parameter + . + + + - If necessary, vacuum can be forced to process all-frozen - pages using the new DISABLE_PAGE_SKIPPING option. - Normally, this should never be needed but it might help in - recovering from visibility-map corruption. + Speed up text sorts where the same string occurs multiple times + (Peter Geoghegan) - Avoid useless heap-truncation attempts during VACUUM - (Jeff Janes, Tom Lane) + Speed up sorting of uuid, bytea, and + char(n) fields by using abbreviated keys + (Peter Geoghegan) - This change avoids taking an exclusive table lock in some cases - where no truncation is possible. The main benefit comes from - avoiding unnecessary query cancellations on standby servers. + Support for abbreviated keys has also been + added to the non-default operator classes text_pattern_ops, + varchar_pattern_ops, and + bpchar_pattern_ops. Processing of ordered-set + aggregates can also now exploit abbreviated keys. + + + + + + + Speed up CREATE INDEX CONCURRENTLY by treating + TIDs as 64-bit integers during sorting (Peter + Geoghegan) + + + + + + + + + Locking + + + + + + + Reduce contention for the ProcArrayLock (Amit Kapila, + Robert Haas) + + + + + + + Improve performance by moving buffer content locks into the buffer + descriptors (Andres Freund, Simon Riggs) + + + + + + + Replace shared-buffer header spinlocks with atomic operations to + improve scalability (Alexander Korotkov, Andres Freund) + + + + + + + Use atomic operations, rather than a spinlock, to protect an + LWLock's wait queue (Andres Freund) + + + + + + + Partition the shared hash table freelist to reduce contention on + multi-CPU-socket servers (Aleksander Alekseev) @@ -531,111 +606,138 @@ This commit is also listed under libpq and psql This change avoids substantial replication delays that sometimes - occurre while replaying such operations. + occurred while replaying such operations. + + + + + + Optimizer Statistics + + + - Avoid computing GROUP BY columns if they are - functionally dependent on other columns (David Rowley) + Improve ANALYZE's estimates for columns with many nulls + (Tomas Vondra, Alex Shulgin) - If a GROUP BY clause includes all columns of a - non-deferred primary key, as well as other columns of the same - table, those other columns are redundant and can be dropped - from the grouping. This saves computation in many common cases. + Previously ANALYZE tended to underestimate the number + of non-NULL distinct values in a column with many + NULLs, and was also inaccurate in computing the + most-common values. - When appropriate, postpone evaluation of SELECT - output expressions until after an ORDER BY sort - (Konstantin Knizhnik) - - - - This change ensures that volatile or expensive functions in the - output list are executed in the order suggested by ORDER - BY, and that they are not evaluated more times than required - when there is a LIMIT clause. Previously, these - properties held if the ordering was performed by an index scan or - pre-merge-join sort, but not if it was performed by a top-level - sort. + Improve planner's estimate of the number of distinct values in + a query result (Tomas Vondra) - Where feasible, trigger kernel writeback after a configurable - number of writes, to prevent accumulation of dirty data in kernel - disk buffers (Fabien Coelho, Andres Freund) + Use foreign key relationships to infer selectivity for join + predicates (Tomas Vondra, David Rowley) - PostgreSQL writes data to the kernel's disk cache, - from where it will be flushed to physical storage in due time. - Many operating systems are not smart about managing this and allow - large amounts of dirty data to accumulate before deciding to flush - it all at once, leading to long delays for new I/O requests. - This change attempts to alleviate this problem by explicitly - requesting data flushes after a configurable interval. + If a table t has a foreign key restriction, say + (a,b) REFERENCES r (x,y), then a WHERE + condition such as t.a = r.x AND t.b = r.y cannot + select more than one r row per t row. + The planner formerly considered AND conditions + to be independent and would often drastically misestimate + selectivity as a result. Now it compares the WHERE + conditions to applicable foreign key constraints and produces + better estimates. + + + + + + + + <command>VACUUM</> + + + + - On Linux, sync_file_range() is used for this purpose, - and the feature is on by default on Linux because that function has few - downsides. This sync capability is also available on other platforms - that have msync() or posix_fadvise(), - but those interfaces have some undesirable side-effects so the - feature is disabled by default on non-Linux platforms. + Avoid re-vacuuming pages containing only frozen tuples (Masahiko + Sawada, Robert Haas, Andres Freund) - The new configuration parameters , , , and control this behavior. + Formerly, anti-wraparound vacuum had to visit every page of + a table, even pages where there was nothing to do. Now, pages + containing only already-frozen tuples are identified in the table's + visibility map, and can be skipped by vacuum even when doing + transaction wraparound prevention. This should greatly reduce the + cost of maintaining large tables containing mostly-unchanged data. + + + + If necessary, vacuum can be forced to process all-frozen + pages using the new DISABLE_PAGE_SKIPPING option. + Normally, this should never be needed but it might help in + recovering from visibility-map corruption. - Perform checkpoint writes in sorted order (Fabien Coelho, - Andres Freund) + Avoid useless heap-truncation attempts during VACUUM + (Jeff Janes, Tom Lane) - Previously, checkpoints wrote out dirty pages in whatever order - they happen to appear in shared buffers, which usually is nearly - random. That performs poorly, especially on rotating media. - This change causes checkpoint-driven writes to be done in order - by file and block number, and to be balanced across tablespaces. + This change avoids taking an exclusive table lock in some cases + where no truncation is possible. The main benefit comes from + avoiding unnecessary query cancellations on standby servers. + + + + + + General Performance + + + + + Avoid computing GROUP BY columns if they are + functionally dependent on other columns (David Rowley) + + + + If a GROUP BY clause includes all columns of a + non-deferred primary key, as well as other columns of the same + table, those other columns are redundant and can be dropped + from the grouping. This saves computation in many common cases. + + + + + + + When appropriate, postpone evaluation of SELECT + output expressions until after an ORDER BY sort + (Konstantin Knizhnik) + + + + This change ensures that volatile or expensive functions in the + output list are executed in the order suggested by ORDER + BY, and that they are not evaluated more times than required + when there is a LIMIT clause. Previously, these + properties held if the ordering was performed by an index scan or + pre-merge-join sort, but not if it was performed by a top-level + sort. + + + + + @@ -687,25 +828,63 @@ This commit is also listed under libpq and psql - Use foreign key relationships to infer selectivity for join - predicates (Tomas Vondra, David Rowley) + Perform checkpoint writes in sorted order (Fabien Coelho, + Andres Freund) - If a table t has a foreign key restriction, say - (a,b) REFERENCES r (x,y), then a WHERE - condition such as t.a = r.x AND t.b = r.y cannot - select more than one r row per t row. - The planner formerly considered AND conditions - to be independent and would often drastically misestimate - selectivity as a result. Now it compares the WHERE - conditions to applicable foreign key constraints and produces - better estimates. + Previously, checkpoints wrote out dirty pages in whatever order + they happen to appear in shared buffers, which usually is nearly + random. That performs poorly, especially on rotating media. + This change causes checkpoint-driven writes to be done in order + by file and block number, and to be balanced across tablespaces. + + + + + + + Where feasible, trigger kernel writeback after a configurable + number of writes, to prevent accumulation of dirty data in kernel + disk buffers (Fabien Coelho, Andres Freund) + + + + PostgreSQL writes data to the kernel's disk cache, + from where it will be flushed to physical storage in due time. + Many operating systems are not smart about managing this and allow + large amounts of dirty data to accumulate before deciding to flush + it all at once, leading to long delays for new I/O requests. + This change attempts to alleviate this problem by explicitly + requesting data flushes after a configurable interval. + + + + On Linux, sync_file_range() is used for this purpose, + and the feature is on by default on Linux because that function has few + downsides. This sync capability is also available on other platforms + that have msync() or posix_fadvise(), + but those interfaces have some undesirable side-effects so the + feature is disabled by default on non-Linux platforms. + + + + The new configuration parameters , , , and control this behavior. @@ -808,35 +987,6 @@ This commit is also listed under libpq and psql - - Improve ANALYZE's estimates for columns with many nulls - (Tomas Vondra, Alex Shulgin) - - - - Previously ANALYZE tended to underestimate the number - of non-NULL distinct values in a column with many - NULLs, and was also inaccurate in computing the - most-common values. - - - - - - - Improve planner's estimate of the number of distinct values in - a query result (Tomas Vondra) - - - - - @@ -851,69 +1001,6 @@ This commit is also listed under libpq and psql - - Improve sorting performance by using quicksort, not replacement - selection sort, when performing external sort steps (Peter - Geoghegan) - - - - The new approach makes better use of the CPU cache - for typical cache sizes and data volumes. Where necessary, - the behavior can be adjusted via the new configuration parameter - . - - - - - - - Speed up text sorts where the same string occurs multiple times - (Peter Geoghegan) - - - - - - - Speed up sorting of uuid, bytea, and - char(n) fields by using abbreviated keys - (Peter Geoghegan) - - - - Support for abbreviated keys has also been - added to the non-default operator classes text_pattern_ops, - varchar_pattern_ops, and - bpchar_pattern_ops. Processing of ordered-set - aggregates can also now exploit abbreviated keys. - - - - - - - Speed up CREATE INDEX CONCURRENTLY by treating - TIDs as 64-bit integers during sorting (Peter - Geoghegan) - - - - - @@ -924,57 +1011,6 @@ This commit is also listed under libpq and psql - - Reduce contention for the ProcArrayLock (Amit Kapila, - Robert Haas) - - - - - - - Improve performance by moving buffer content locks into the buffer - descriptors (Andres Freund, Simon Riggs) - - - - - - - Replace shared-buffer header spinlocks with atomic operations to - improve scalability (Alexander Korotkov, Andres Freund) - - - - - - - Use atomic operations, rather than a spinlock, to protect an - LWLock's wait queue (Andres Freund) - - - - - - - Partition the shared hash table freelist to reduce contention on - multi-CPU-socket servers (Aleksander Alekseev) - - - - - @@ -1017,6 +1053,21 @@ This commit is also listed under libpq and psql + + Add pg_control_system(), + pg_control_checkpoint(), + pg_control_recovery(), and + pg_control_init() functions to expose fields of + pg_control to SQL (Joe Conway, Michael + Paquier) + + + + + @@ -1082,21 +1133,6 @@ This commit is also listed under libpq and psql - - Add pg_control_system(), - pg_control_checkpoint(), - pg_control_recovery(), and - pg_control_init() functions to expose fields of - pg_control to SQL (Joe Conway, Michael - Paquier) - - - - - @@ -1219,18 +1255,18 @@ This commit is also listed under libpq and psql - Add configure option - This allows the use of systemd service units of - type notify, which greatly simplifies the management - of PostgreSQL under systemd. + This behavior is controlled by the new configuration parameter + . It can + be useful to prevent forgotten transactions from holding locks + or preventing vacuum cleanup for too long. @@ -1247,18 +1283,18 @@ This commit is also listed under libpq and psql - Allow sessions to be terminated automatically if they are in - idle-in-transaction state for too long (Vik Fearing) + Add configure option - This behavior is controlled by the new configuration parameter - . It can - be useful to prevent forgotten transactions from holding locks - or preventing vacuum cleanup for too long. + This allows the use of systemd service units of + type notify, which greatly simplifies the management + of PostgreSQL under systemd. @@ -1857,6 +1893,30 @@ XXX this is pending backpatch, may need to remove + + Improve full-text search to support searching for phrases, that + is, lexemes appearing adjacent to each other in a specific order, + or with a specified distance between them (Teodor Sigaev, Oleg + Bartunov, Dmitry Ivanov) + + + + A phrase-search query can be specified in tsquery + input using the new operators <-> and + <N>. The former means + that the lexemes before and after it must appear adjacent to + each other in that order. The latter means they must be exactly + N lexemes apart. + + + + + @@ -1917,30 +1977,6 @@ XXX this is pending backpatch, may need to remove - - Improve full-text search to support searching for phrases, that - is, lexemes appearing adjacent to each other in a specific order, - or with a specified distance between them (Teodor Sigaev, Oleg - Bartunov, Dmitry Ivanov) - - - - A phrase-search query can be specified in tsquery - input using the new operators <-> and - <N>. The former means - that the lexemes before and after it must appear adjacent to - each other in that order. The latter means they must be exactly - N lexemes apart. - - - - - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/src/sgml/stylesheet-speedup-xhtml.xsl b/doc/src/sgml/stylesheet-speedup-xhtml.xsl new file mode 100644 index 0000000000..8428dc5870 --- /dev/null +++ b/doc/src/sgml/stylesheet-speedup-xhtml.xsl @@ -0,0 +1,252 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Error: If you change $chunk.section.depth, then you must update the performance-optimized chunk-all-sections-template. + + + + + + + + + + + + + + + diff --git a/doc/src/sgml/stylesheet.xsl b/doc/src/sgml/stylesheet.xsl index 7967b361dd..631fcc4edf 100644 --- a/doc/src/sgml/stylesheet.xsl +++ b/doc/src/sgml/stylesheet.xsl @@ -6,6 +6,7 @@ + From 5697522d8411135d26a5d807f9e4afa182502f64 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 18 Aug 2016 14:48:51 -0400 Subject: [PATCH 031/871] In plpgsql, don't try to convert int2vector or oidvector to expanded array. These types are storage-compatible with real arrays, but they don't support toasting, so of course they can't support expansion either. Per bug #14289 from Michael Overmeyer. Back-patch to 9.5 where expanded arrays were introduced. Report: <20160818174414.1529.37913@wrigleys.postgresql.org> --- src/include/utils/array.h | 2 +- src/pl/plpgsql/src/pl_comp.c | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/include/utils/array.h b/src/include/utils/array.h index b62b08c482..6164f119ba 100644 --- a/src/include/utils/array.h +++ b/src/include/utils/array.h @@ -36,7 +36,7 @@ * * The OIDVECTOR and INT2VECTOR datatypes are storage-compatible with * generic arrays, but they support only one-dimensional arrays with no - * nulls (and no null bitmap). + * nulls (and no null bitmap). They don't support being toasted, either. * * There are also some "fixed-length array" datatypes, such as NAME and * POINT. These are simply a sequence of a fixed number of items each diff --git a/src/pl/plpgsql/src/pl_comp.c b/src/pl/plpgsql/src/pl_comp.c index b628c2811b..38aa030303 100644 --- a/src/pl/plpgsql/src/pl_comp.c +++ b/src/pl/plpgsql/src/pl_comp.c @@ -2192,14 +2192,19 @@ build_datatype(HeapTuple typeTup, int32 typmod, Oid collation) /* NB: this is only used to decide whether to apply expand_array */ if (typeStruct->typtype == TYPTYPE_BASE) { - /* this test should match what get_element_type() checks */ + /* + * This test should include what get_element_type() checks. We also + * disallow non-toastable array types (i.e. oidvector and int2vector). + */ typ->typisarray = (typeStruct->typlen == -1 && - OidIsValid(typeStruct->typelem)); + OidIsValid(typeStruct->typelem) && + typeStruct->typstorage != 'p'); } else if (typeStruct->typtype == TYPTYPE_DOMAIN) { /* we can short-circuit looking up base types if it's not varlena */ typ->typisarray = (typeStruct->typlen == -1 && + typeStruct->typstorage != 'p' && OidIsValid(get_base_element_type(typeStruct->typbasetype))); } else From c5d4f40cb5e231eb2cbc533b5f094f3a4829e2ef Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 18 Aug 2016 16:04:35 -0400 Subject: [PATCH 032/871] Update line count totals for psql help displays. As usual, we've been pretty awful about maintaining these counts. They're not all that critical, perhaps, but let's get them right at release time. Also fix 9.5, which I notice is just as bad. It's probably wrong further back, but the lack of --help=foo options before 9.5 makes it too painful to count. --- src/bin/psql/help.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c index efc845414f..a69c4dd8cf 100644 --- a/src/bin/psql/help.c +++ b/src/bin/psql/help.c @@ -69,7 +69,7 @@ usage(unsigned short int pager) * Keep this line count in sync with the number of lines printed below! * Use "psql --help=options | wc" to count correctly. */ - output = PageOutput(60, pager ? &(pset.popt.topt) : NULL); + output = PageOutput(61, pager ? &(pset.popt.topt) : NULL); fprintf(output, _("psql is the PostgreSQL interactive terminal.\n\n")); fprintf(output, _("Usage:\n")); @@ -168,7 +168,7 @@ slashUsage(unsigned short int pager) * Use "psql --help=commands | wc" to count correctly. It's okay to count * the USE_READLINE line even in builds without that. */ - output = PageOutput(111, pager ? &(pset.popt.topt) : NULL); + output = PageOutput(113, pager ? &(pset.popt.topt) : NULL); fprintf(output, _("General\n")); fprintf(output, _(" \\copyright show PostgreSQL usage and distribution terms\n")); @@ -325,7 +325,7 @@ helpVariables(unsigned short int pager) * Windows builds currently print one more line than non-Windows builds. * Using the larger number is fine. */ - output = PageOutput(87, pager ? &(pset.popt.topt) : NULL); + output = PageOutput(88, pager ? &(pset.popt.topt) : NULL); fprintf(output, _("List of specially treated variables\n\n")); From 1d2e73a3dfdbd7168b323fa39879c60df6076412 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 18 Aug 2016 12:00:00 -0400 Subject: [PATCH 033/871] Remove obsolete replacement system() on darwin Per comment in the file, this was fixed around OS X 10.2. --- src/backend/port/Makefile | 4 -- src/backend/port/darwin/Makefile | 17 ----- src/backend/port/darwin/README | 36 ----------- src/backend/port/darwin/system.c | 104 ------------------------------- 4 files changed, 161 deletions(-) delete mode 100644 src/backend/port/darwin/Makefile delete mode 100644 src/backend/port/darwin/README delete mode 100644 src/backend/port/darwin/system.c diff --git a/src/backend/port/Makefile b/src/backend/port/Makefile index 89549d0d2b..aba1e92fe1 100644 --- a/src/backend/port/Makefile +++ b/src/backend/port/Makefile @@ -23,9 +23,6 @@ include $(top_builddir)/src/Makefile.global OBJS = atomics.o dynloader.o pg_sema.o pg_shmem.o $(TAS) -ifeq ($(PORTNAME), darwin) -SUBDIRS += darwin -endif ifeq ($(PORTNAME), win32) SUBDIRS += win32 endif @@ -44,5 +41,4 @@ endif distclean clean: rm -f tas_cpp.s - $(MAKE) -C darwin clean $(MAKE) -C win32 clean diff --git a/src/backend/port/darwin/Makefile b/src/backend/port/darwin/Makefile deleted file mode 100644 index 9d463ffb87..0000000000 --- a/src/backend/port/darwin/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -#------------------------------------------------------------------------- -# -# Makefile-- -# Makefile for port/darwin -# -# IDENTIFICATION -# src/backend/port/darwin/Makefile -# -#------------------------------------------------------------------------- - -subdir = src/backend/port/darwin -top_builddir = ../../../.. -include $(top_builddir)/src/Makefile.global - -OBJS = system.o - -include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/port/darwin/README b/src/backend/port/darwin/README deleted file mode 100644 index 2d9df79683..0000000000 --- a/src/backend/port/darwin/README +++ /dev/null @@ -1,36 +0,0 @@ -src/backend/port/darwin/README - -Darwin -====== - -The file system.c included herein is taken directly from Apple's Darwin -open-source CVS archives, and is redistributed under the BSD copyright -notice it bears. (According to Apple's CVS logs, their version is -identical to the FreeBSD original.) It provides our own implementation of -the system(3) function, which ought by all rights to be identical to the -one provided in libc on Darwin machines. Nonetheless, this version works, -whereas the one that actually ships with Mac OS X 10.1 doesn't. The -shipped version appears to disconnect the calling process from any shared -memory segments it is attached to. (The symptom seen in PostgreSQL is -that a backend attempting to execute CREATE DATABASE core-dumps.) I would -love to know why there is a discrepancy between the published source and -the actual behavior --- tgl 7-Nov-2001. - -Appropriate bug reports have been filed with Apple --- see -Radar Bug#s 2767956, 2683531, 2805147. One hopes we can retire this -kluge in the not too distant future. - - -As of PostgreSQL 7.3 and Mac OS X 10.1, one should expect warnings -like these while linking the backend: - -/usr/bin/ld: warning unused multiple definitions of symbol _system -port/SUBSYS.o definition of _system in section (__TEXT,__text) -/usr/lib/libm.dylib(system.o) unused definition of _system - -These are due to overriding system() per the above-described hack. - - -The bug appears to be repaired in OS X 10.2.6 and later (possibly in -earlier 10.2.* as well, but no systems handy to check). We #ifdef out -the substitute system() definition on 10.3 and later. diff --git a/src/backend/port/darwin/system.c b/src/backend/port/darwin/system.c deleted file mode 100644 index 1cd5266929..0000000000 --- a/src/backend/port/darwin/system.c +++ /dev/null @@ -1,104 +0,0 @@ -/* - * src/backend/port/darwin/system.c - * - * only needed in OS X 10.1 and possibly early 10.2 releases */ -#include /* pgrminclude ignore */ -#if MAC_OS_X_VERSION_MAX_ALLOWED <= MAC_OS_X_VERSION_10_2 || !defined(MAC_OS_X_VERSION_10_2) - -/* - * Copyright (c) 1988, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD: src/lib/libc/stdlib/system.c,v 1.6 2000/03/16 02:14:41 jasone Exp $ - */ - -#if defined(LIBC_SCCS) && !defined(lint) -static char sccsid[] = "@(#)system.c 8.1 (Berkeley) 6/4/93"; -#endif /* LIBC_SCCS and not lint */ - -#include -#include -#include -#include -#include - -int system(const char *command); - -int -system(const char *command) -{ - pid_t pid; - int pstat; - struct sigaction ign, - intact, - quitact; - sigset_t newsigblock, - oldsigblock; - - if (!command) /* just checking... */ - return (1); - - /* - * Ignore SIGINT and SIGQUIT, block SIGCHLD. Remember to save existing - * signal dispositions. - */ - ign.sa_handler = SIG_IGN; - (void) sigemptyset(&ign.sa_mask); - ign.sa_flags = 0; - (void) sigaction(SIGINT, &ign, &intact); - (void) sigaction(SIGQUIT, &ign, &quitact); - (void) sigemptyset(&newsigblock); - (void) sigaddset(&newsigblock, SIGCHLD); - (void) sigprocmask(SIG_BLOCK, &newsigblock, &oldsigblock); - switch (pid = fork()) - { - case -1: /* error */ - break; - case 0: /* child */ - - /* - * Restore original signal dispositions and exec the command. - */ - (void) sigaction(SIGINT, &intact, NULL); - (void) sigaction(SIGQUIT, &quitact, NULL); - (void) sigprocmask(SIG_SETMASK, &oldsigblock, NULL); - execl(_PATH_BSHELL, "sh", "-c", command, (char *) NULL); - _exit(127); - default: /* parent */ - do - { - pid = wait4(pid, &pstat, 0, (struct rusage *) 0); - } while (pid == -1 && errno == EINTR); - break; - } - (void) sigaction(SIGINT, &intact, NULL); - (void) sigaction(SIGQUIT, &quitact, NULL); - (void) sigprocmask(SIG_SETMASK, &oldsigblock, NULL); - return (pid == -1 ? -1 : pstat); -} - -#endif /* OS X < 10.3 */ From 9595383bc6fc24d25970374e2eddd5ce6f977f9e Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 18 Aug 2016 17:30:14 -0700 Subject: [PATCH 034/871] Add alternative output for ON CONFLICT toast isolation test. On some buildfarm animals the isolationtest added in 07ef0351 failed, as the order in which processes are run after unlocking is not guaranteed. Add an alternative output for that. Discussion: <7969.1471484738@sss.pgh.pa.us> Backpatch: 9.6, like the test in the aforementioned commit --- .../expected/insert-conflict-toast_1.out | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/test/isolation/expected/insert-conflict-toast_1.out diff --git a/src/test/isolation/expected/insert-conflict-toast_1.out b/src/test/isolation/expected/insert-conflict-toast_1.out new file mode 100644 index 0000000000..18346162b7 --- /dev/null +++ b/src/test/isolation/expected/insert-conflict-toast_1.out @@ -0,0 +1,15 @@ +Parsed test spec with 3 sessions + +starting permutation: s2insert s3insert s1commit +pg_advisory_xact_lock + + +step s2insert: + INSERT INTO ctoast (key, val) VALUES (1, ctoast_large_val()) ON CONFLICT DO NOTHING; + +step s3insert: + INSERT INTO ctoast (key, val) VALUES (1, ctoast_large_val()) ON CONFLICT DO NOTHING; + +step s1commit: COMMIT; +step s3insert: <... completed> +step s2insert: <... completed> From 5285c5e873d8b622da7007c1628e5afa80f372fb Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 18 Aug 2016 21:41:10 -0400 Subject: [PATCH 035/871] doc: requirepeer is a way to avoid spoofing We already mentioned unix_socket_directories as an option. Reported-by: https://www.postgresql.org/message-id/45016837-6cf3-3136-f959-763d06a28076%402ndquadrant.com Backpatch-through: 9.6 --- doc/src/sgml/runtime.sgml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 60a06590fe..98752c2875 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1922,7 +1922,7 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 - The simplest way to prevent spoofing for local + On way to prevent spoofing of local connections is to use a Unix domain socket directory () that has write permission only for a trusted local user. This prevents a malicious user from creating @@ -1934,6 +1934,13 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 /tmp cleanup script to prevent removal of the symbolic link. + + Another option for local connections is for clients to use + requirepeer + to specify the required owner of the server process connected to + the socket. + + To prevent spoofing on TCP connections, the best solution is to use SSL certificates and make sure that clients check the server's certificate. From 6eefd2422ef232aec2fe12465d9ec4018c63814d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 19 Aug 2016 12:51:02 -0400 Subject: [PATCH 036/871] Remove typedef celt from the regex library, along with macro NOCELT. The regex library used to have a notion of a "collating element" that was distinct from a "character", but Henry Spencer never actually implemented his planned support for multi-character collating elements, and the Tcl crew ripped out most of the stubs for that years ago. The only thing left that distinguished the "celt" typedef from the "chr" typedef was that "celt" was supposed to also be able to hold the not-a-character "NOCELT" value. However, NOCELT was not used anywhere after the MCCE stub removal changes, which means there's no need for celt to be different from chr. Removing the separate typedef simplifies matters and also removes a trap for the unwary, in that celt is signed while chr may not be, so comparisons could mean different things. There's no bug there today because we restrict CHR_MAX to be less than INT_MAX, but I think there may have been such bugs before we did that, and there could be again if anyone ever decides to fool with the range of chr. This patch also removes assorted unnecessary casts to "chr" of values that are already chrs. Many of these seem to be leftover from days when the code was compatible with pre-ANSI C. --- src/backend/regex/regc_cvec.c | 6 +++--- src/backend/regex/regc_lex.c | 12 +++++------ src/backend/regex/regc_locale.c | 37 ++++++++++++++++----------------- src/backend/regex/regcomp.c | 12 +++++------ src/include/regex/regcustom.h | 4 +--- 5 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/backend/regex/regc_cvec.c b/src/backend/regex/regc_cvec.c index 921a7d7f92..3a9e8cfbbd 100644 --- a/src/backend/regex/regc_cvec.c +++ b/src/backend/regex/regc_cvec.c @@ -78,7 +78,7 @@ addchr(struct cvec * cv, /* character vector */ chr c) /* character to add */ { assert(cv->nchrs < cv->chrspace); - cv->chrs[cv->nchrs++] = (chr) c; + cv->chrs[cv->nchrs++] = c; } /* @@ -90,8 +90,8 @@ addrange(struct cvec * cv, /* character vector */ chr to) /* last character of range */ { assert(cv->nranges < cv->rangespace); - cv->ranges[cv->nranges * 2] = (chr) from; - cv->ranges[cv->nranges * 2 + 1] = (chr) to; + cv->ranges[cv->nranges * 2] = from; + cv->ranges[cv->nranges * 2 + 1] = to; cv->nranges++; } diff --git a/src/backend/regex/regc_lex.c b/src/backend/regex/regc_lex.c index f62ec7dc81..cd34c8ae41 100644 --- a/src/backend/regex/regc_lex.c +++ b/src/backend/regex/regc_lex.c @@ -870,7 +870,7 @@ lexescape(struct vars * v) if (v->now == save || ((int) c > 0 && (int) c <= v->nsubexp)) { NOTE(REG_UBACKREF); - RETV(BACKREF, (chr) c); + RETV(BACKREF, c); } /* oops, doesn't look like it's a backref after all... */ v->now = save; @@ -986,10 +986,8 @@ lexdigits(struct vars * v, */ static int /* 1 normal, 0 failure */ brenext(struct vars * v, - chr pc) + chr c) { - chr c = (chr) pc; - switch (c) { case CHR('*'): @@ -1153,7 +1151,7 @@ chrnamed(struct vars * v, const chr *endp, /* just past end of name */ chr lastresort) /* what to return if name lookup fails */ { - celt c; + chr c; int errsave; int e; struct cvec *cv; @@ -1165,10 +1163,10 @@ chrnamed(struct vars * v, v->err = errsave; if (e != 0) - return (chr) lastresort; + return lastresort; cv = range(v, c, c, 0); if (cv->nchrs == 0) - return (chr) lastresort; + return lastresort; return cv->chrs[0]; } diff --git a/src/backend/regex/regc_locale.c b/src/backend/regex/regc_locale.c index 4fe62921e3..399de027cd 100644 --- a/src/backend/regex/regc_locale.c +++ b/src/backend/regex/regc_locale.c @@ -361,9 +361,9 @@ static const struct cname /* - * element - map collating-element name to celt + * element - map collating-element name to chr */ -static celt +static chr element(struct vars * v, /* context */ const chr *startp, /* points to start of name */ const chr *endp) /* points just past end of name */ @@ -401,13 +401,13 @@ element(struct vars * v, /* context */ */ static struct cvec * range(struct vars * v, /* context */ - celt a, /* range start */ - celt b, /* range end, might equal a */ + chr a, /* range start */ + chr b, /* range end, might equal a */ int cases) /* case-independent? */ { int nchrs; struct cvec *cv; - celt c, + chr c, cc; if (a != b && !before(a, b)) @@ -444,7 +444,7 @@ range(struct vars * v, /* context */ for (c = a; c <= b; c++) { - cc = pg_wc_tolower((chr) c); + cc = pg_wc_tolower(c); if (cc != c && (before(cc, a) || before(b, cc))) { @@ -455,7 +455,7 @@ range(struct vars * v, /* context */ } addchr(cv, cc); } - cc = pg_wc_toupper((chr) c); + cc = pg_wc_toupper(c); if (cc != c && (before(cc, a) || before(b, cc))) { @@ -477,10 +477,10 @@ range(struct vars * v, /* context */ } /* - * before - is celt x before celt y, for purposes of range legality? + * before - is chr x before chr y, for purposes of range legality? */ static int /* predicate */ -before(celt x, celt y) +before(chr x, chr y) { if (x < y) return 1; @@ -493,7 +493,7 @@ before(celt x, celt y) */ static struct cvec * eclass(struct vars * v, /* context */ - celt c, /* Collating element representing the + chr c, /* Collating element representing the * equivalence class. */ int cases) /* all cases? */ { @@ -503,12 +503,12 @@ eclass(struct vars * v, /* context */ if ((v->cflags & REG_FAKE) && c == 'x') { cv = getcvec(v, 4, 0); - addchr(cv, (chr) 'x'); - addchr(cv, (chr) 'y'); + addchr(cv, CHR('x')); + addchr(cv, CHR('y')); if (cases) { - addchr(cv, (chr) 'X'); - addchr(cv, (chr) 'Y'); + addchr(cv, CHR('X')); + addchr(cv, CHR('Y')); } return cv; } @@ -518,7 +518,7 @@ eclass(struct vars * v, /* context */ return allcases(v, c); cv = getcvec(v, 1, 0); assert(cv != NULL); - addchr(cv, (chr) c); + addchr(cv, c); return cv; } @@ -673,15 +673,14 @@ cclass(struct vars * v, /* context */ */ static struct cvec * allcases(struct vars * v, /* context */ - chr pc) /* character to get case equivs of */ + chr c) /* character to get case equivs of */ { struct cvec *cv; - chr c = (chr) pc; chr lc, uc; - lc = pg_wc_tolower((chr) c); - uc = pg_wc_toupper((chr) c); + lc = pg_wc_tolower(c); + uc = pg_wc_toupper(c); cv = getcvec(v, 2, 0); addchr(cv, lc); diff --git a/src/backend/regex/regcomp.c b/src/backend/regex/regcomp.c index cc589b0930..48d63da11d 100644 --- a/src/backend/regex/regcomp.c +++ b/src/backend/regex/regcomp.c @@ -210,10 +210,10 @@ static pg_wchar pg_wc_toupper(pg_wchar c); static pg_wchar pg_wc_tolower(pg_wchar c); /* === regc_locale.c === */ -static celt element(struct vars *, const chr *, const chr *); -static struct cvec *range(struct vars *, celt, celt, int); -static int before(celt, celt); -static struct cvec *eclass(struct vars *, celt, int); +static chr element(struct vars *, const chr *, const chr *); +static struct cvec *range(struct vars *, chr, chr, int); +static int before(chr, chr); +static struct cvec *eclass(struct vars *, chr, int); static struct cvec *cclass(struct vars *, const chr *, const chr *, int); static struct cvec *allcases(struct vars *, chr); static int cmp(const chr *, const chr *, size_t); @@ -1424,8 +1424,8 @@ brackpart(struct vars * v, struct state * lp, struct state * rp) { - celt startc; - celt endc; + chr startc; + chr endc; struct cvec *cv; const chr *startp; const chr *endp; diff --git a/src/include/regex/regcustom.h b/src/include/regex/regcustom.h index 60034daee8..459851a7f6 100644 --- a/src/include/regex/regcustom.h +++ b/src/include/regex/regcustom.h @@ -58,15 +58,13 @@ /* internal character type and related */ typedef pg_wchar chr; /* the type itself */ typedef unsigned uchr; /* unsigned type that will hold a chr */ -typedef int celt; /* type to hold chr, or NOCELT */ -#define NOCELT (-1) /* celt value which is not valid chr */ #define CHR(c) ((unsigned char) (c)) /* turn char literal into chr literal */ #define DIGITVAL(c) ((c)-'0') /* turn chr digit into its value */ #define CHRBITS 32 /* bits in a chr; must not use sizeof */ #define CHR_MIN 0x00000000 /* smallest and largest chr; the value */ #define CHR_MAX 0x7ffffffe /* CHR_MAX-CHR_MIN+1 must fit in an int, and - * CHR_MAX+1 must fit in both chr and celt */ + * CHR_MAX+1 must fit in a chr variable */ /* * Check if a chr value is in range. Ideally we'd just write this as From a859e640035680db31531ccd19a67292dd726baf Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 19 Aug 2016 13:31:10 -0400 Subject: [PATCH 037/871] Clean up another pre-ANSI-C-ism in regex code: get rid of pcolor typedef. pcolor was used to represent function arguments that are nominally of type color, but when using a pre-ANSI C compiler would be passed as the promoted integer type. We really don't need that anymore. --- src/backend/regex/regc_color.c | 10 +++++----- src/backend/regex/regc_nfa.c | 8 ++++---- src/backend/regex/regcomp.c | 14 +++++++------- src/backend/regex/rege_dfa.c | 6 +++--- src/backend/regex/regexec.c | 4 ++-- src/include/regex/regguts.h | 1 - 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/backend/regex/regc_color.c b/src/backend/regex/regc_color.c index c495cee300..8ffc8fb797 100644 --- a/src/backend/regex/regc_color.c +++ b/src/backend/regex/regc_color.c @@ -148,7 +148,7 @@ cmtreefree(struct colormap * cm, static color /* previous color */ setcolor(struct colormap * cm, chr c, - pcolor co) + color co) { uchr uc = c; int shift; @@ -199,7 +199,7 @@ setcolor(struct colormap * cm, b = uc & BYTMASK; prev = t->tcolor[b]; - t->tcolor[b] = (color) co; + t->tcolor[b] = co; return prev; } @@ -293,7 +293,7 @@ newcolor(struct colormap * cm) */ static void freecolor(struct colormap * cm, - pcolor co) + color co) { struct colordesc *cd = &cm->cd[co]; color pco, @@ -393,7 +393,7 @@ subcolor(struct colormap * cm, chr c) */ static color newsub(struct colormap * cm, - pcolor co) + color co) { color sco; /* new subcolor */ @@ -658,7 +658,7 @@ static void rainbow(struct nfa * nfa, struct colormap * cm, int type, - pcolor but, /* COLORLESS if no exceptions */ + color but, /* COLORLESS if no exceptions */ struct state * from, struct state * to) { diff --git a/src/backend/regex/regc_nfa.c b/src/backend/regex/regc_nfa.c index cd9a3239bd..90dca5d9de 100644 --- a/src/backend/regex/regc_nfa.c +++ b/src/backend/regex/regc_nfa.c @@ -275,7 +275,7 @@ destroystate(struct nfa * nfa, static void newarc(struct nfa * nfa, int t, - pcolor co, + color co, struct state * from, struct state * to) { @@ -321,7 +321,7 @@ newarc(struct nfa * nfa, static void createarc(struct nfa * nfa, int t, - pcolor co, + color co, struct state * from, struct state * to) { @@ -334,7 +334,7 @@ createarc(struct nfa * nfa, assert(a != NULL); a->type = t; - a->co = (color) co; + a->co = co; a->to = to; a->from = from; @@ -553,7 +553,7 @@ hasnonemptyout(struct state * s) static struct arc * findarc(struct state * s, int type, - pcolor co) + color co) { struct arc *a; diff --git a/src/backend/regex/regcomp.c b/src/backend/regex/regcomp.c index 48d63da11d..b211cc0a18 100644 --- a/src/backend/regex/regcomp.c +++ b/src/backend/regex/regcomp.c @@ -97,19 +97,19 @@ static chr chrnamed(struct vars *, const chr *, const chr *, chr); static void initcm(struct vars *, struct colormap *); static void freecm(struct colormap *); static void cmtreefree(struct colormap *, union tree *, int); -static color setcolor(struct colormap *, chr, pcolor); +static color setcolor(struct colormap *, chr, color); static color maxcolor(struct colormap *); static color newcolor(struct colormap *); -static void freecolor(struct colormap *, pcolor); +static void freecolor(struct colormap *, color); static color pseudocolor(struct colormap *); static color subcolor(struct colormap *, chr c); -static color newsub(struct colormap *, pcolor); +static color newsub(struct colormap *, color); static void subrange(struct vars *, chr, chr, struct state *, struct state *); static void subblock(struct vars *, chr, struct state *, struct state *); static void okcolors(struct nfa *, struct colormap *); static void colorchain(struct colormap *, struct arc *); static void uncolorchain(struct colormap *, struct arc *); -static void rainbow(struct nfa *, struct colormap *, int, pcolor, struct state *, struct state *); +static void rainbow(struct nfa *, struct colormap *, int, color, struct state *, struct state *); static void colorcomplement(struct nfa *, struct colormap *, int, struct state *, struct state *, struct state *); #ifdef REG_DEBUG @@ -125,13 +125,13 @@ static struct state *newfstate(struct nfa *, int flag); static void dropstate(struct nfa *, struct state *); static void freestate(struct nfa *, struct state *); static void destroystate(struct nfa *, struct state *); -static void newarc(struct nfa *, int, pcolor, struct state *, struct state *); -static void createarc(struct nfa *, int, pcolor, struct state *, struct state *); +static void newarc(struct nfa *, int, color, struct state *, struct state *); +static void createarc(struct nfa *, int, color, struct state *, struct state *); static struct arc *allocarc(struct nfa *, struct state *); static void freearc(struct nfa *, struct arc *); static void changearctarget(struct arc *, struct state *); static int hasnonemptyout(struct state *); -static struct arc *findarc(struct state *, int, pcolor); +static struct arc *findarc(struct state *, int, color); static void cparc(struct nfa *, struct arc *, struct state *, struct state *); static void sortins(struct nfa *, struct state *); static int sortins_cmp(const void *, const void *); diff --git a/src/backend/regex/rege_dfa.c b/src/backend/regex/rege_dfa.c index 7d90242ace..b98c9d3902 100644 --- a/src/backend/regex/rege_dfa.c +++ b/src/backend/regex/rege_dfa.c @@ -603,7 +603,7 @@ static struct sset * miss(struct vars * v, struct dfa * d, struct sset * css, - pcolor co, + color co, chr *cp, /* next chr */ chr *start) /* where the attempt got started */ { @@ -731,7 +731,7 @@ miss(struct vars * v, css->outs[co] = p; css->inchain[co] = p->ins; p->ins.ss = css; - p->ins.co = (color) co; + p->ins.co = co; } return p; } @@ -743,7 +743,7 @@ static int /* predicate: constraint satisfied? */ lacon(struct vars * v, struct cnfa * pcnfa, /* parent cnfa */ chr *cp, - pcolor co) /* "color" of the lookaround constraint */ + color co) /* "color" of the lookaround constraint */ { int n; struct subre *sub; diff --git a/src/backend/regex/regexec.c b/src/backend/regex/regexec.c index 82659a0f2f..5cbfd9b151 100644 --- a/src/backend/regex/regexec.c +++ b/src/backend/regex/regexec.c @@ -159,8 +159,8 @@ static struct dfa *newdfa(struct vars *, struct cnfa *, struct colormap *, struc static void freedfa(struct dfa *); static unsigned hash(unsigned *, int); static struct sset *initialize(struct vars *, struct dfa *, chr *); -static struct sset *miss(struct vars *, struct dfa *, struct sset *, pcolor, chr *, chr *); -static int lacon(struct vars *, struct cnfa *, chr *, pcolor); +static struct sset *miss(struct vars *, struct dfa *, struct sset *, color, chr *, chr *); +static int lacon(struct vars *, struct cnfa *, chr *, color); static struct sset *getvacant(struct vars *, struct dfa *, chr *, chr *); static struct sset *pickss(struct vars *, struct dfa *, chr *, chr *); diff --git a/src/include/regex/regguts.h b/src/include/regex/regguts.h index 2ceffa6563..b0aa641cc4 100644 --- a/src/include/regex/regguts.h +++ b/src/include/regex/regguts.h @@ -149,7 +149,6 @@ * which are of much more manageable number. */ typedef short color; /* colors of characters */ -typedef int pcolor; /* what color promotes to */ #define MAX_COLOR 32767 /* max color (must fit in 'color' datatype) */ #define COLORLESS (-1) /* impossible color */ From 6f79ae7fe549bed8bbd1f54ddd9b98f8f9a315f5 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 19 Aug 2016 14:38:55 -0300 Subject: [PATCH 038/871] reorderbuffer: preserve errno while reporting error Clobbering errno during cleanup after an error is an oft-repeated, easy to make mistake. Deal with it here as everywhere else, by saving it aside and restoring after cleanup, before ereport'ing. In passing, add a missing errcode declaration in another ereport() call in the same file, which I noticed while skimming the file looking for similar problems. Backpatch to 9.4, where this code was introduced. --- src/backend/replication/logical/reorderbuffer.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 9594b1c671..213ce34674 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -2317,7 +2317,10 @@ ReorderBufferSerializeChange(ReorderBuffer *rb, ReorderBufferTXN *txn, if (write(fd, rb->outbuf, ondisk->size) != ondisk->size) { + int save_errno = errno; + CloseTransientFile(fd); + errno = save_errno; ereport(ERROR, (errcode_for_file_access(), errmsg("could not write to data file for XID %u: %m", @@ -3070,7 +3073,8 @@ ApplyLogicalMappingFile(HTAB *tuplecid_data, Oid relid, const char *fname) fd = OpenTransientFile(path, O_RDONLY | PG_BINARY, 0); if (fd < 0) ereport(ERROR, - (errmsg("could not open file \"%s\": %m", path))); + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); while (true) { From da1c91631e3577ea5818f855ebb5bd206d559006 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 19 Aug 2016 14:03:07 -0400 Subject: [PATCH 039/871] Speed up planner's scanning for parallel-query hazards. We need to scan the whole parse tree for parallel-unsafe functions. If there are none, we'll later need to determine whether particular subtrees contain any parallel-restricted functions. The previous coding retained no knowledge from the first scan, even though this is very wasteful in the common case where the query contains only parallel-safe functions. We can bypass all of the later scans by remembering that fact. This provides a small but measurable speed improvement when the case applies, and shouldn't cost anything when it doesn't. Patch by me, reviewed by Robert Haas Discussion: <3740.1471538387@sss.pgh.pa.us> --- src/backend/nodes/outfuncs.c | 1 + src/backend/optimizer/path/allpaths.c | 30 +------ src/backend/optimizer/plan/planmain.c | 9 +- src/backend/optimizer/plan/planner.c | 43 ++++++---- src/backend/optimizer/util/clauses.c | 115 +++++++++++++++++++------- src/backend/optimizer/util/pathnode.c | 6 +- src/backend/optimizer/util/relnode.c | 4 +- src/include/nodes/relation.h | 2 + src/include/optimizer/clauses.h | 3 +- 9 files changed, 133 insertions(+), 80 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 1fab807772..50019f4164 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2029,6 +2029,7 @@ _outPlannerGlobal(StringInfo str, const PlannerGlobal *node) WRITE_BOOL_FIELD(dependsOnRole); WRITE_BOOL_FIELD(parallelModeOK); WRITE_BOOL_FIELD(parallelModeNeeded); + WRITE_CHAR_FIELD(maxParallelHazard); } static void diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 88d833a2e8..af73792227 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -78,7 +78,6 @@ static void set_plain_rel_size(PlannerInfo *root, RelOptInfo *rel, static void create_plain_partial_paths(PlannerInfo *root, RelOptInfo *rel); static void set_rel_consider_parallel(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); -static bool function_rte_parallel_ok(RangeTblEntry *rte); static void set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static void set_tablesample_rel_size(PlannerInfo *root, RelOptInfo *rel, @@ -542,8 +541,7 @@ set_rel_consider_parallel(PlannerInfo *root, RelOptInfo *rel, if (proparallel != PROPARALLEL_SAFE) return; - if (has_parallel_hazard((Node *) rte->tablesample->args, - false)) + if (!is_parallel_safe(root, (Node *) rte->tablesample->args)) return; } @@ -596,7 +594,7 @@ set_rel_consider_parallel(PlannerInfo *root, RelOptInfo *rel, case RTE_FUNCTION: /* Check for parallel-restricted functions. */ - if (!function_rte_parallel_ok(rte)) + if (!is_parallel_safe(root, (Node *) rte->functions)) return; break; @@ -629,40 +627,20 @@ set_rel_consider_parallel(PlannerInfo *root, RelOptInfo *rel, * outer join clauses work correctly. It would likely break equivalence * classes, too. */ - if (has_parallel_hazard((Node *) rel->baserestrictinfo, false)) + if (!is_parallel_safe(root, (Node *) rel->baserestrictinfo)) return; /* * Likewise, if the relation's outputs are not parallel-safe, give up. * (Usually, they're just Vars, but sometimes they're not.) */ - if (has_parallel_hazard((Node *) rel->reltarget->exprs, false)) + if (!is_parallel_safe(root, (Node *) rel->reltarget->exprs)) return; /* We have a winner. */ rel->consider_parallel = true; } -/* - * Check whether a function RTE is scanning something parallel-restricted. - */ -static bool -function_rte_parallel_ok(RangeTblEntry *rte) -{ - ListCell *lc; - - foreach(lc, rte->functions) - { - RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); - - Assert(IsA(rtfunc, RangeTblFunction)); - if (has_parallel_hazard(rtfunc->funcexpr, false)) - return false; - } - - return true; -} - /* * set_plain_rel_pathlist * Build access paths for a plain relation (no subquery, no inheritance) diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 27234ffa22..e7ae7ae8ae 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -71,14 +71,13 @@ query_planner(PlannerInfo *root, List *tlist, /* * If query allows parallelism in general, check whether the quals are - * parallel-restricted. There's currently no real benefit to setting - * this flag correctly because we can't yet reference subplans from - * parallel workers. But that might change someday, so set this - * correctly anyway. + * parallel-restricted. (We need not check final_rel->reltarget + * because it's empty at this point. Anything parallel-restricted in + * the query tlist will be dealt with later.) */ if (root->glob->parallelModeOK) final_rel->consider_parallel = - !has_parallel_hazard(parse->jointree->quals, false); + is_parallel_safe(root, parse->jointree->quals); /* The only path for it is a trivial Result path */ add_path(final_rel, (Path *) diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index b265628325..174210be6c 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -23,6 +23,7 @@ #include "access/sysattr.h" #include "access/xact.h" #include "catalog/pg_constraint_fn.h" +#include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "executor/executor.h" #include "executor/nodeAgg.h" @@ -241,12 +242,26 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) * time and execution time, so don't generate a parallel plan if we're in * serializable mode. */ - glob->parallelModeOK = (cursorOptions & CURSOR_OPT_PARALLEL_OK) != 0 && - IsUnderPostmaster && dynamic_shared_memory_type != DSM_IMPL_NONE && - parse->commandType == CMD_SELECT && !parse->hasModifyingCTE && - parse->utilityStmt == NULL && max_parallel_workers_per_gather > 0 && - !IsParallelWorker() && !IsolationIsSerializable() && - !has_parallel_hazard((Node *) parse, true); + if ((cursorOptions & CURSOR_OPT_PARALLEL_OK) != 0 && + IsUnderPostmaster && + dynamic_shared_memory_type != DSM_IMPL_NONE && + parse->commandType == CMD_SELECT && + parse->utilityStmt == NULL && + !parse->hasModifyingCTE && + max_parallel_workers_per_gather > 0 && + !IsParallelWorker() && + !IsolationIsSerializable()) + { + /* all the cheap tests pass, so scan the query tree */ + glob->maxParallelHazard = max_parallel_hazard(parse); + glob->parallelModeOK = (glob->maxParallelHazard != PROPARALLEL_UNSAFE); + } + else + { + /* skip the query tree scan, just assume it's unsafe */ + glob->maxParallelHazard = PROPARALLEL_UNSAFE; + glob->parallelModeOK = false; + } /* * glob->parallelModeNeeded should tell us whether it's necessary to @@ -1802,7 +1817,7 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, * computed by partial paths. */ if (current_rel->partial_pathlist && - !has_parallel_hazard((Node *) scanjoin_target->exprs, false)) + is_parallel_safe(root, (Node *) scanjoin_target->exprs)) { /* Apply the scan/join target to each partial path */ foreach(lc, current_rel->partial_pathlist) @@ -1948,8 +1963,8 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, * query. */ if (current_rel->consider_parallel && - !has_parallel_hazard(parse->limitOffset, false) && - !has_parallel_hazard(parse->limitCount, false)) + is_parallel_safe(root, parse->limitOffset) && + is_parallel_safe(root, parse->limitCount)) final_rel->consider_parallel = true; /* @@ -3326,8 +3341,8 @@ create_grouping_paths(PlannerInfo *root, * target list and HAVING quals are parallel-safe. */ if (input_rel->consider_parallel && - !has_parallel_hazard((Node *) target->exprs, false) && - !has_parallel_hazard((Node *) parse->havingQual, false)) + is_parallel_safe(root, (Node *) target->exprs) && + is_parallel_safe(root, (Node *) parse->havingQual)) grouped_rel->consider_parallel = true; /* @@ -3881,8 +3896,8 @@ create_window_paths(PlannerInfo *root, * target list and active windows for non-parallel-safe constructs. */ if (input_rel->consider_parallel && - !has_parallel_hazard((Node *) output_target->exprs, false) && - !has_parallel_hazard((Node *) activeWindows, false)) + is_parallel_safe(root, (Node *) output_target->exprs) && + is_parallel_safe(root, (Node *) activeWindows)) window_rel->consider_parallel = true; /* @@ -4272,7 +4287,7 @@ create_ordered_paths(PlannerInfo *root, * target list is parallel-safe. */ if (input_rel->consider_parallel && - !has_parallel_hazard((Node *) target->exprs, false)) + is_parallel_safe(root, (Node *) target->exprs)) ordered_rel->consider_parallel = true; /* diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index a40ad40606..4496fde056 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -91,8 +91,9 @@ typedef struct typedef struct { - bool allow_restricted; -} has_parallel_hazard_arg; + char max_hazard; /* worst proparallel hazard found so far */ + char max_interesting; /* worst proparallel hazard of interest */ +} max_parallel_hazard_context; static bool contain_agg_clause_walker(Node *node, void *context); static bool get_agg_clause_costs_walker(Node *node, @@ -103,8 +104,8 @@ static bool contain_subplans_walker(Node *node, void *context); static bool contain_mutable_functions_walker(Node *node, void *context); static bool contain_volatile_functions_walker(Node *node, void *context); static bool contain_volatile_functions_not_nextval_walker(Node *node, void *context); -static bool has_parallel_hazard_walker(Node *node, - has_parallel_hazard_arg *context); +static bool max_parallel_hazard_walker(Node *node, + max_parallel_hazard_context *context); static bool contain_nonstrict_functions_walker(Node *node, void *context); static bool contain_context_dependent_node(Node *clause); static bool contain_context_dependent_node_walker(Node *node, int *flags); @@ -1100,46 +1101,98 @@ contain_volatile_functions_not_nextval_walker(Node *node, void *context) context); } + /***************************************************************************** * Check queries for parallel unsafe and/or restricted constructs *****************************************************************************/ /* - * Check whether a node tree contains parallel hazards. This is used both on - * the entire query tree, to see whether the query can be parallelized at all - * (with allow_restricted = true), and also to evaluate whether a particular - * expression is safe to run within a parallel worker (with allow_restricted = - * false). We could separate these concerns into two different functions, but - * there's enough overlap that it doesn't seem worthwhile. + * max_parallel_hazard + * Find the worst parallel-hazard level in the given query + * + * Returns the worst function hazard property (the earliest in this list: + * PROPARALLEL_UNSAFE, PROPARALLEL_RESTRICTED, PROPARALLEL_SAFE) that can + * be found in the given parsetree. We use this to find out whether the query + * can be parallelized at all. The caller will also save the result in + * PlannerGlobal so as to short-circuit checks of portions of the querytree + * later, in the common case where everything is SAFE. + */ +char +max_parallel_hazard(Query *parse) +{ + max_parallel_hazard_context context; + + context.max_hazard = PROPARALLEL_SAFE; + context.max_interesting = PROPARALLEL_UNSAFE; + (void) max_parallel_hazard_walker((Node *) parse, &context); + return context.max_hazard; +} + +/* + * is_parallel_safe + * Detect whether the given expr contains only parallel-safe functions + * + * root->glob->maxParallelHazard must previously have been set to the + * result of max_parallel_hazard() on the whole query. */ bool -has_parallel_hazard(Node *node, bool allow_restricted) +is_parallel_safe(PlannerInfo *root, Node *node) { - has_parallel_hazard_arg context; + max_parallel_hazard_context context; - context.allow_restricted = allow_restricted; - return has_parallel_hazard_walker(node, &context); + /* If max_parallel_hazard found nothing unsafe, we don't need to look */ + if (root->glob->maxParallelHazard == PROPARALLEL_SAFE) + return true; + /* Else use max_parallel_hazard's search logic, but stop on RESTRICTED */ + context.max_hazard = PROPARALLEL_SAFE; + context.max_interesting = PROPARALLEL_RESTRICTED; + return !max_parallel_hazard_walker(node, &context); } +/* core logic for all parallel-hazard checks */ static bool -has_parallel_hazard_checker(Oid func_id, void *context) +max_parallel_hazard_test(char proparallel, max_parallel_hazard_context *context) { - char proparallel = func_parallel(func_id); + switch (proparallel) + { + case PROPARALLEL_SAFE: + /* nothing to see here, move along */ + break; + case PROPARALLEL_RESTRICTED: + /* increase max_hazard to RESTRICTED */ + Assert(context->max_hazard != PROPARALLEL_UNSAFE); + context->max_hazard = proparallel; + /* done if we are not expecting any unsafe functions */ + if (context->max_interesting == proparallel) + return true; + break; + case PROPARALLEL_UNSAFE: + context->max_hazard = proparallel; + /* we're always done at the first unsafe construct */ + return true; + default: + elog(ERROR, "unrecognized proparallel value \"%c\"", proparallel); + break; + } + return false; +} - if (((has_parallel_hazard_arg *) context)->allow_restricted) - return (proparallel == PROPARALLEL_UNSAFE); - else - return (proparallel != PROPARALLEL_SAFE); +/* check_functions_in_node callback */ +static bool +max_parallel_hazard_checker(Oid func_id, void *context) +{ + return max_parallel_hazard_test(func_parallel(func_id), + (max_parallel_hazard_context *) context); } static bool -has_parallel_hazard_walker(Node *node, has_parallel_hazard_arg *context) +max_parallel_hazard_walker(Node *node, max_parallel_hazard_context *context) { if (node == NULL) return false; /* Check for hazardous functions in node itself */ - if (check_functions_in_node(node, has_parallel_hazard_checker, + if (check_functions_in_node(node, max_parallel_hazard_checker, context)) return true; @@ -1156,7 +1209,7 @@ has_parallel_hazard_walker(Node *node, has_parallel_hazard_arg *context) */ if (IsA(node, CoerceToDomain)) { - if (!context->allow_restricted) + if (max_parallel_hazard_test(PROPARALLEL_RESTRICTED, context)) return true; } @@ -1167,7 +1220,7 @@ has_parallel_hazard_walker(Node *node, has_parallel_hazard_arg *context) { RestrictInfo *rinfo = (RestrictInfo *) node; - return has_parallel_hazard_walker((Node *) rinfo->clause, context); + return max_parallel_hazard_walker((Node *) rinfo->clause, context); } /* @@ -1176,13 +1229,13 @@ has_parallel_hazard_walker(Node *node, has_parallel_hazard_arg *context) * not worry about examining their contents; if they are unsafe, we would * have found that out while examining the whole tree before reduction of * sublinks to subplans. (Really we should not see SubLink during a - * not-allow_restricted scan, but if we do, return true.) + * max_interesting == restricted scan, but if we do, return true.) */ else if (IsA(node, SubLink) || IsA(node, SubPlan) || IsA(node, AlternativeSubPlan)) { - if (!context->allow_restricted) + if (max_parallel_hazard_test(PROPARALLEL_RESTRICTED, context)) return true; } @@ -1192,7 +1245,7 @@ has_parallel_hazard_walker(Node *node, has_parallel_hazard_arg *context) */ else if (IsA(node, Param)) { - if (!context->allow_restricted) + if (max_parallel_hazard_test(PROPARALLEL_RESTRICTED, context)) return true; } @@ -1207,20 +1260,24 @@ has_parallel_hazard_walker(Node *node, has_parallel_hazard_arg *context) /* SELECT FOR UPDATE/SHARE must be treated as unsafe */ if (query->rowMarks != NULL) + { + context->max_hazard = PROPARALLEL_UNSAFE; return true; + } /* Recurse into subselects */ return query_tree_walker(query, - has_parallel_hazard_walker, + max_parallel_hazard_walker, context, 0); } /* Recurse to check arguments */ return expression_tree_walker(node, - has_parallel_hazard_walker, + max_parallel_hazard_walker, context); } + /***************************************************************************** * Check clauses for nonstrict functions *****************************************************************************/ diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index ce7ad545a9..abb7507d8e 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -2178,7 +2178,7 @@ create_projection_path(PlannerInfo *root, pathnode->path.parallel_aware = false; pathnode->path.parallel_safe = rel->consider_parallel && subpath->parallel_safe && - !has_parallel_hazard((Node *) target->exprs, false); + is_parallel_safe(root, (Node *) target->exprs); pathnode->path.parallel_workers = subpath->parallel_workers; /* Projection does not change the sort order */ pathnode->path.pathkeys = subpath->pathkeys; @@ -2285,7 +2285,7 @@ apply_projection_to_path(PlannerInfo *root, * target expressions, then we can't. */ if (IsA(path, GatherPath) && - !has_parallel_hazard((Node *) target->exprs, false)) + is_parallel_safe(root, (Node *) target->exprs)) { GatherPath *gpath = (GatherPath *) path; @@ -2306,7 +2306,7 @@ apply_projection_to_path(PlannerInfo *root, target); } else if (path->parallel_safe && - has_parallel_hazard((Node *) target->exprs, false)) + !is_parallel_safe(root, (Node *) target->exprs)) { /* * We're inserting a parallel-restricted target list into a path diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 806600ed10..deef5605b7 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -513,8 +513,8 @@ build_join_rel(PlannerInfo *root, * here. */ if (inner_rel->consider_parallel && outer_rel->consider_parallel && - !has_parallel_hazard((Node *) restrictlist, false) && - !has_parallel_hazard((Node *) joinrel->reltarget->exprs, false)) + is_parallel_safe(root, (Node *) restrictlist) && + is_parallel_safe(root, (Node *) joinrel->reltarget->exprs)) joinrel->consider_parallel = true; /* diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 2be8908445..fcfb0d4d0f 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -126,6 +126,8 @@ typedef struct PlannerGlobal bool parallelModeOK; /* parallel mode potentially OK? */ bool parallelModeNeeded; /* parallel mode actually required? */ + + char maxParallelHazard; /* worst PROPARALLEL hazard level */ } PlannerGlobal; /* macro for fetching the Plan associated with a SubPlan node */ diff --git a/src/include/optimizer/clauses.h b/src/include/optimizer/clauses.h index be7c639f7b..9abef37cb6 100644 --- a/src/include/optimizer/clauses.h +++ b/src/include/optimizer/clauses.h @@ -61,7 +61,8 @@ extern bool contain_subplans(Node *clause); extern bool contain_mutable_functions(Node *clause); extern bool contain_volatile_functions(Node *clause); extern bool contain_volatile_functions_not_nextval(Node *clause); -extern bool has_parallel_hazard(Node *node, bool allow_restricted); +extern char max_parallel_hazard(Query *parse); +extern bool is_parallel_safe(PlannerInfo *root, Node *node); extern bool contain_nonstrict_functions(Node *clause); extern bool contain_leaked_vars(Node *clause); From 65a603e90328a7a8fb3ab30ed96f24bf8eb4cf84 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 19 Aug 2016 14:35:32 -0400 Subject: [PATCH 040/871] Guard against parallel-restricted functions in VALUES expressions. Obvious brain fade in set_rel_consider_parallel(). Noticed it while adjusting the adjacent RTE_FUNCTION case. In 9.6, also make the code look more like what I just did in HEAD by removing the unnecessary function_rte_parallel_ok subroutine (it does nothing that expression_tree_walker wouldn't do). --- src/backend/optimizer/path/allpaths.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index af73792227..04264b4335 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -599,11 +599,9 @@ set_rel_consider_parallel(PlannerInfo *root, RelOptInfo *rel, break; case RTE_VALUES: - - /* - * The data for a VALUES clause is stored in the plan tree itself, - * so scanning it in a worker is fine. - */ + /* Check for parallel-restricted functions. */ + if (!is_parallel_safe(root, (Node *) rte->values_lists)) + return; break; case RTE_CTE: From 8299471c37fff0b0f5a777a12f920125310c0efe Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 19 Aug 2016 17:13:47 -0400 Subject: [PATCH 041/871] Use LEFT JOINs in some system views in case referenced row doesn't exist. In particular, left join to pg_authid so that rows in pg_stat_activity don't disappear if the session's owning user has been dropped. Also convert a few joins to pg_database to left joins, in the same spirit, though that case might be harder to hit. We were doing this in other views already, so it was a bit inconsistent that these views didn't. Oskari Saarenmaa, with some further tweaking by me Discussion: <56E87CD8.60007@ohmu.fi> --- src/backend/catalog/system_views.sql | 20 +++++++++----------- src/include/catalog/catversion.h | 2 +- src/test/regress/expected/rules.out | 20 +++++++++----------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 4fc5d5a065..ada214274f 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -642,9 +642,9 @@ CREATE VIEW pg_stat_activity AS S.backend_xid, s.backend_xmin, S.query - FROM pg_database D, pg_stat_get_activity(NULL) AS S, pg_authid U - WHERE S.datid = D.oid AND - S.usesysid = U.oid; + FROM pg_stat_get_activity(NULL) AS S + LEFT JOIN pg_database AS D ON (S.datid = D.oid) + LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid); CREATE VIEW pg_stat_replication AS SELECT @@ -664,10 +664,9 @@ CREATE VIEW pg_stat_replication AS W.replay_location, W.sync_priority, W.sync_state - FROM pg_stat_get_activity(NULL) AS S, pg_authid U, - pg_stat_get_wal_senders() AS W - WHERE S.usesysid = U.oid AND - S.pid = W.pid; + FROM pg_stat_get_activity(NULL) AS S + JOIN pg_stat_get_wal_senders() AS W ON (S.pid = W.pid) + LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid); CREATE VIEW pg_stat_wal_receiver AS SELECT @@ -813,7 +812,7 @@ CREATE VIEW pg_stat_progress_vacuum AS S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count, S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples FROM pg_stat_get_progress_info('VACUUM') AS S - JOIN pg_database D ON S.datid = D.oid; + LEFT JOIN pg_database D ON S.datid = D.oid; CREATE VIEW pg_user_mappings AS SELECT @@ -832,12 +831,11 @@ CREATE VIEW pg_user_mappings AS NULL END AS umoptions FROM pg_user_mapping U - LEFT JOIN pg_authid A ON (A.oid = U.umuser) JOIN - pg_foreign_server S ON (U.umserver = S.oid); + JOIN pg_foreign_server S ON (U.umserver = S.oid) + LEFT JOIN pg_authid A ON (A.oid = U.umuser); REVOKE ALL on pg_user_mapping FROM public; - CREATE VIEW pg_replication_origin_status AS SELECT * FROM pg_show_replication_origin_status(); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index fb356bf3cd..26f6126002 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201608171 +#define CATALOG_VERSION_NO 201608191 #endif diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 8157324fee..00700f28dc 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1664,10 +1664,9 @@ pg_stat_activity| SELECT s.datid, s.backend_xid, s.backend_xmin, s.query - FROM pg_database d, - pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn), - pg_authid u - WHERE ((s.datid = d.oid) AND (s.usesysid = u.oid)); + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn) + LEFT JOIN pg_database d ON ((s.datid = d.oid))) + LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_all_indexes| SELECT c.oid AS relid, i.oid AS indexrelid, n.nspname AS schemaname, @@ -1776,7 +1775,7 @@ pg_stat_progress_vacuum| SELECT s.pid, s.param6 AS max_dead_tuples, s.param7 AS num_dead_tuples FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10) - JOIN pg_database d ON ((s.datid = d.oid))); + LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_replication| SELECT s.pid, s.usesysid, u.rolname AS usename, @@ -1793,10 +1792,9 @@ pg_stat_replication| SELECT s.pid, w.replay_location, w.sync_priority, w.sync_state - FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn), - pg_authid u, - pg_stat_get_wal_senders() w(pid, state, sent_location, write_location, flush_location, replay_location, sync_priority, sync_state) - WHERE ((s.usesysid = u.oid) AND (s.pid = w.pid)); + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn) + JOIN pg_stat_get_wal_senders() w(pid, state, sent_location, write_location, flush_location, replay_location, sync_priority, sync_state) ON ((s.pid = w.pid))) + LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_ssl| SELECT s.pid, s.ssl, s.sslversion AS version, @@ -2155,8 +2153,8 @@ pg_user_mappings| SELECT u.oid AS umid, ELSE NULL::text[] END AS umoptions FROM ((pg_user_mapping u - LEFT JOIN pg_authid a ON ((a.oid = u.umuser))) - JOIN pg_foreign_server s ON ((u.umserver = s.oid))); + JOIN pg_foreign_server s ON ((u.umserver = s.oid))) + LEFT JOIN pg_authid a ON ((a.oid = u.umuser))); pg_views| SELECT n.nspname AS schemaname, c.relname AS viewname, pg_get_userbyid(c.relowner) AS viewowner, From 6471045230f5d891ad724c54d406e2214f3c96d9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 19 Aug 2016 17:32:59 -0400 Subject: [PATCH 042/871] Allow empty queries in pgbench. This might have been too much of a foot-gun before 9.6, but with the new commands-end-at-semicolons parsing rule, the only way to get an empty query into a script is to explicitly write an extra ";". So we may as well allow the case. Fabien Coelho Patch: --- src/bin/pgbench/pgbench.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 87fb006d87..8027955121 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -1898,6 +1898,7 @@ doCustom(TState *thread, CState *st, StatsData *agg) { case PGRES_COMMAND_OK: case PGRES_TUPLES_OK: + case PGRES_EMPTY_QUERY: break; /* OK */ default: fprintf(stderr, "client %d aborted in state %d: %s", From a00c58314745772f6c6a49b6d02a9572cd600bda Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 20 Aug 2016 15:05:25 -0400 Subject: [PATCH 043/871] Make initdb's suggested "pg_ctl start" command line more reliable. The original coding here was not nearly careful enough about quoting special characters, and it didn't get corner cases right for constructing the pg_ctl path either. Use join_path_components() and appendShellString() to do it honestly, so that the string will more likely work if blindly copied-and-pasted. While at it, teach appendShellString() not to quote strings that clearly don't need it, so that the output from initdb doesn't become uglier than it was before in typical cases where quoting is not needed. Ryan Murphy, reviewed by Michael Paquier and myself Discussion: --- src/bin/initdb/Makefile | 3 +++ src/bin/initdb/initdb.c | 43 ++++++++++++++++++++++++------------- src/fe_utils/string_utils.c | 17 +++++++++++++-- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/src/bin/initdb/Makefile b/src/bin/initdb/Makefile index 094c8945c9..531cc979a4 100644 --- a/src/bin/initdb/Makefile +++ b/src/bin/initdb/Makefile @@ -18,6 +18,9 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) -I$(top_srcdir)/src/timezone $(CPPFLAGS) +# note: we need libpq only because fe_utils does +LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) + # use system timezone data? ifneq (,$(with_system_tzdata)) override CPPFLAGS += '-DSYSTEMTZDIR="$(with_system_tzdata)"' diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index a978bbc328..aad6ba5639 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -67,6 +67,7 @@ #include "getaddrinfo.h" #include "getopt_long.h" #include "miscadmin.h" +#include "fe_utils/string_utils.h" /* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */ @@ -331,14 +332,6 @@ do { \ output_failed = true, output_errno = errno; \ } while (0) -#ifndef WIN32 -#define QUOTE_PATH "" -#define DIR_SEP "/" -#else -#define QUOTE_PATH "\"" -#define DIR_SEP "\\" -#endif - static char * escape_quotes(const char *src) { @@ -3359,7 +3352,8 @@ main(int argc, char *argv[]) int c; int option_index; char *effective_user; - char bin_dir[MAXPGPATH]; + PQExpBuffer start_db_cmd; + char pg_ctl_path[MAXPGPATH]; /* * Ensure that buffering behavior of stdout and stderr matches what it is @@ -3587,14 +3581,33 @@ main(int argc, char *argv[]) if (authwarning != NULL) fprintf(stderr, "%s", authwarning); - /* Get directory specification used to start this executable */ - strlcpy(bin_dir, argv[0], sizeof(bin_dir)); - get_parent_directory(bin_dir); + /* + * Build up a shell command to tell the user how to start the server + */ + start_db_cmd = createPQExpBuffer(); + + /* Get directory specification used to start initdb ... */ + strlcpy(pg_ctl_path, argv[0], sizeof(pg_ctl_path)); + canonicalize_path(pg_ctl_path); + get_parent_directory(pg_ctl_path); + /* ... and tag on pg_ctl instead */ + join_path_components(pg_ctl_path, pg_ctl_path, "pg_ctl"); + + /* path to pg_ctl, properly quoted */ + appendShellString(start_db_cmd, pg_ctl_path); + + /* add -D switch, with properly quoted data directory */ + appendPQExpBufferStr(start_db_cmd, " -D "); + appendShellString(start_db_cmd, pgdata_native); + + /* add suggested -l switch and "start" command */ + appendPQExpBufferStr(start_db_cmd, " -l logfile start"); printf(_("\nSuccess. You can now start the database server using:\n\n" - " %s%s%spg_ctl%s -D %s%s%s -l logfile start\n\n"), - QUOTE_PATH, bin_dir, (strlen(bin_dir) > 0) ? DIR_SEP : "", QUOTE_PATH, - QUOTE_PATH, pgdata_native, QUOTE_PATH); + " %s\n\n"), + start_db_cmd->data); + + destroyPQExpBuffer(start_db_cmd); return 0; } diff --git a/src/fe_utils/string_utils.c b/src/fe_utils/string_utils.c index 2c566b1ad7..edbc869e45 100644 --- a/src/fe_utils/string_utils.c +++ b/src/fe_utils/string_utils.c @@ -418,7 +418,7 @@ appendByteaLiteral(PQExpBuffer buf, const unsigned char *str, size_t length, /* * Append the given string to the shell command being built in the buffer, - * with suitable shell-style quoting to create exactly one argument. + * with shell-style quoting as needed to create exactly one argument. * * Forbid LF or CR characters, which have scant practical use beyond designing * security breaches. The Windows command shell is unusable as a conduit for @@ -429,8 +429,22 @@ appendByteaLiteral(PQExpBuffer buf, const unsigned char *str, size_t length, void appendShellString(PQExpBuffer buf, const char *str) { +#ifdef WIN32 + int backslash_run_length = 0; +#endif const char *p; + /* + * Don't bother with adding quotes if the string is nonempty and clearly + * contains only safe characters. + */ + if (*str != '\0' && + strspn(str, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./:") == strlen(str)) + { + appendPQExpBufferStr(buf, str); + return; + } + #ifndef WIN32 appendPQExpBufferChar(buf, '\''); for (p = str; *p; p++) @@ -450,7 +464,6 @@ appendShellString(PQExpBuffer buf, const char *str) } appendPQExpBufferChar(buf, '\''); #else /* WIN32 */ - int backslash_run_length = 0; /* * A Windows system() argument experiences two layers of interpretation. From 04164deb7cb8e572302e2b43786fa24de3c40da3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 20 Aug 2016 16:53:25 -0400 Subject: [PATCH 044/871] initdb now needs to reference libpq include files in MSVC builds. Fallout from commit a00c58314. Per buildfarm. --- src/tools/msvc/Mkvcbuild.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index da4d9847fc..6746728616 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -50,7 +50,7 @@ my @contrib_excludes = ( # Set of variables for frontend modules my $frontend_defines = { 'initdb' => 'FRONTEND' }; -my @frontend_uselibpq = ('pg_ctl', 'pg_upgrade', 'pgbench', 'psql'); +my @frontend_uselibpq = ('pg_ctl', 'pg_upgrade', 'pgbench', 'psql', 'initdb'); my @frontend_uselibpgport = ( 'pg_archivecleanup', 'pg_test_fsync', 'pg_test_timing', 'pg_upgrade', From 9132c014290d02435999c81892fa8b0b384497d8 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Sun, 21 Aug 2016 22:05:57 -0400 Subject: [PATCH 045/871] Retire escapeConnectionParameter(). It is redundant with appendConnStrVal(), which became an extern function in commit 41f18f021a0882eccbeca62e2ed4b66c6b96e9c9. This changes the handling of out-of-memory and of certain inputs for which quoting is optional, but pg_basebackup has no need for unusual treatment thereof. --- src/bin/pg_basebackup/Makefile | 1 + src/bin/pg_basebackup/pg_basebackup.c | 69 ++------------------------- 2 files changed, 4 insertions(+), 66 deletions(-) diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile index 585467205b..a23a83eb9b 100644 --- a/src/bin/pg_basebackup/Makefile +++ b/src/bin/pg_basebackup/Makefile @@ -17,6 +17,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) +LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpq OBJS=receivelog.o streamutil.o $(WIN32RES) diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index ed41db8e6e..351a42068f 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -26,6 +26,7 @@ #endif #include "common/string.h" +#include "fe_utils/string_utils.h" #include "getopt_long.h" #include "libpq-fe.h" #include "pqexpbuffer.h" @@ -1392,69 +1393,6 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) WriteRecoveryConf(); } -/* - * Escape a parameter value so that it can be used as part of a libpq - * connection string, e.g. in: - * - * application_name= - * - * The returned string is malloc'd. Return NULL on out-of-memory. - */ -static char * -escapeConnectionParameter(const char *src) -{ - bool need_quotes = false; - bool need_escaping = false; - const char *p; - char *dstbuf; - char *dst; - - /* - * First check if quoting is needed. Any quote (') or backslash (\) - * characters need to be escaped. Parameters are separated by whitespace, - * so any string containing whitespace characters need to be quoted. An - * empty string is represented by ''. - */ - if (strchr(src, '\'') != NULL || strchr(src, '\\') != NULL) - need_escaping = true; - - for (p = src; *p; p++) - { - if (isspace((unsigned char) *p)) - { - need_quotes = true; - break; - } - } - - if (*src == '\0') - return pg_strdup("''"); - - if (!need_quotes && !need_escaping) - return pg_strdup(src); /* no quoting or escaping needed */ - - /* - * Allocate a buffer large enough for the worst case that all the source - * characters need to be escaped, plus quotes. - */ - dstbuf = pg_malloc(strlen(src) * 2 + 2 + 1); - - dst = dstbuf; - if (need_quotes) - *(dst++) = '\''; - for (; *src; src++) - { - if (*src == '\'' || *src == '\\') - *(dst++) = '\\'; - *(dst++) = *src; - } - if (need_quotes) - *(dst++) = '\''; - *dst = '\0'; - - return dstbuf; -} - /* * Escape a string so that it can be used as a value in a key-value pair * a configuration file. @@ -1523,9 +1461,8 @@ GenerateRecoveryConf(PGconn *conn) * Write "keyword=value" pieces, the value string is escaped and/or * quoted if necessary. */ - escaped = escapeConnectionParameter(option->val); - appendPQExpBuffer(&conninfo_buf, "%s=%s", option->keyword, escaped); - free(escaped); + appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword); + appendConnStrVal(&conninfo_buf, option->val); } /* From 234309fa87739f7a3ac99de815d181b50f2542e7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 22 Aug 2016 08:01:12 -0400 Subject: [PATCH 046/871] initdb now needs submake-libpq and submake-libpgfeutils. More fallout from commit a00c58314. Pointed out by Michael Paquier. --- src/bin/initdb/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/initdb/Makefile b/src/bin/initdb/Makefile index 531cc979a4..394eae0875 100644 --- a/src/bin/initdb/Makefile +++ b/src/bin/initdb/Makefile @@ -30,7 +30,7 @@ OBJS= initdb.o findtimezone.o localtime.o encnames.o $(WIN32RES) all: initdb -initdb: $(OBJS) | submake-libpgport +initdb: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) # We used to pull in all of libpq to get encnames.c, but that From f9472d72561d285e8c138f3e1276f3110f55e515 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 22 Aug 2016 12:00:00 -0400 Subject: [PATCH 047/871] Run select_parallel test by itself Remove the plpgsql wrapping that hides the context. So now the test will fail if the work doesn't actually happen in a parallel worker. Run the test in its own test group to ensure it won't run out of resources for that. --- src/test/regress/expected/select_parallel.out | 14 ++++---------- src/test/regress/parallel_schedule | 5 ++++- src/test/regress/sql/select_parallel.sql | 10 ++-------- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out index 2286fafab3..18e21b7f13 100644 --- a/src/test/regress/expected/select_parallel.out +++ b/src/test/regress/expected/select_parallel.out @@ -111,14 +111,8 @@ explain (costs off) Index Cond: (unique1 = 1) (5 rows) -do $$begin - -- Provoke error, possibly in worker. If this error happens to occur in - -- the worker, there will be a CONTEXT line which must be hidden. - perform stringu1::int2 from tenk1 where unique1 = 1; - exception - when others then - raise 'SQLERRM: %', sqlerrm; -end$$; -ERROR: SQLERRM: invalid input syntax for integer: "BAAAAA" -CONTEXT: PL/pgSQL function inline_code_block line 7 at RAISE +-- provoke error in worker +select stringu1::int2 from tenk1 where unique1 = 1; +ERROR: invalid input syntax for integer: "BAAAAA" +CONTEXT: parallel worker rollback; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 3815182fe7..1cb5dfc336 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -92,7 +92,10 @@ test: brin gin gist spgist privileges init_privs security_label collate matview test: alter_generic alter_operator misc psql async dbsize misc_functions # rules cannot run concurrently with any test that creates a view -test: rules psql_crosstab select_parallel amutils +test: rules psql_crosstab amutils + +# run by itself so it can run parallel workers +test: select_parallel # ---------- # Another group of parallel tests diff --git a/src/test/regress/sql/select_parallel.sql b/src/test/regress/sql/select_parallel.sql index 38d3166742..8b4090f2ec 100644 --- a/src/test/regress/sql/select_parallel.sql +++ b/src/test/regress/sql/select_parallel.sql @@ -44,13 +44,7 @@ set force_parallel_mode=1; explain (costs off) select stringu1::int2 from tenk1 where unique1 = 1; -do $$begin - -- Provoke error, possibly in worker. If this error happens to occur in - -- the worker, there will be a CONTEXT line which must be hidden. - perform stringu1::int2 from tenk1 where unique1 = 1; - exception - when others then - raise 'SQLERRM: %', sqlerrm; -end$$; +-- provoke error in worker +select stringu1::int2 from tenk1 where unique1 = 1; rollback; From af5743851d7d526fadfeb9726e2b3d8b1fc5026d Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 22 Aug 2016 11:52:43 -0400 Subject: [PATCH 048/871] Improve header comment for LockHasWaitersRelation. Dimitry Ivanov spotted a typo, and I added a bit of wordsmithing. --- src/backend/storage/lmgr/lmgr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c index 7b08555b07..eeedc38251 100644 --- a/src/backend/storage/lmgr/lmgr.c +++ b/src/backend/storage/lmgr/lmgr.c @@ -268,8 +268,8 @@ UnlockRelation(Relation relation, LOCKMODE lockmode) /* * LockHasWaitersRelation * - * This is a functiion to check if someone else is waiting on a - * lock, we are currently holding. + * This is a function to check whether someone else is waiting for a + * lock which we are currently holding. */ bool LockHasWaitersRelation(Relation relation, LOCKMODE lockmode) From 008c4135ccf67e74239a17a85f912d1a51b6349e Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 22 Aug 2016 15:22:11 -0400 Subject: [PATCH 049/871] Fix possible sorting error when aborting use of abbreviated keys. Due to an error in the abbreviated key abort logic, the most recently processed SortTuple could be incorrectly marked NULL, resulting in an incorrect final sort order. In the worst case, this could result in a corrupt btree index, which would need to be rebuild using REINDEX. However, abbrevation doesn't abort very often, not all data types use it, and only one tuple would end up in the wrong place, so the practical impact of this mistake may be somewhat limited. Report and patch by Peter Geoghegan. --- src/backend/utils/sort/tuplesort.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index 510565c339..ae384a8546 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -1443,7 +1443,7 @@ tuplesort_putindextuplevalues(Tuplesortstate *state, Relation rel, mtup->datum1 = index_getattr(tuple, 1, RelationGetDescr(state->indexRel), - &stup.isnull1); + &mtup->isnull1); } } @@ -4271,7 +4271,7 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup) mtup->datum1 = heap_getattr(tuple, state->indexInfo->ii_KeyAttrNumbers[0], state->tupDesc, - &stup->isnull1); + &mtup->isnull1); } } } @@ -4588,7 +4588,7 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup) mtup->datum1 = index_getattr(tuple, 1, RelationGetDescr(state->indexRel), - &stup->isnull1); + &mtup->isnull1); } } } From f2e016f8d55ee029c5d6ec853ff6802aaf49fb0a Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 22 Aug 2016 17:20:44 -0400 Subject: [PATCH 050/871] doc: fix typo in recent patch Reported-by: Jeff Janes Backpatch-through: 9.6 --- doc/src/sgml/runtime.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 98752c2875..ef0139c365 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1922,7 +1922,7 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 - On way to prevent spoofing of local + One way to prevent spoofing of local connections is to use a Unix domain socket directory () that has write permission only for a trusted local user. This prevents a malicious user from creating From 7b405b3e04779fc0a026c9c6ac3e06194948b253 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 23 Aug 2016 09:39:54 -0400 Subject: [PATCH 051/871] Refactor some network.c code to create cidr_set_masklen_internal(). Merge several copies of "copy an inet value and adjust the mask length" code to create a single, conveniently C-callable function. This function is exported for future use by inet SPGiST support, but it's good cleanup anyway since we had three slightly-different-for-no-good-reason copies. (Extracted from a larger patch, to separate new code from refactoring of old code) Emre Hasegeli --- src/backend/utils/adt/network.c | 109 +++++++++----------------------- src/include/utils/inet.h | 5 +- 2 files changed, 33 insertions(+), 81 deletions(-) diff --git a/src/backend/utils/adt/network.c b/src/backend/utils/adt/network.c index 1f8469a2cb..3f6987af04 100644 --- a/src/backend/utils/adt/network.c +++ b/src/backend/utils/adt/network.c @@ -268,11 +268,7 @@ Datum inet_to_cidr(PG_FUNCTION_ARGS) { inet *src = PG_GETARG_INET_PP(0); - inet *dst; int bits; - int byte; - int nbits; - int maxbytes; bits = ip_bits(src); @@ -280,29 +276,7 @@ inet_to_cidr(PG_FUNCTION_ARGS) if ((bits < 0) || (bits > ip_maxbits(src))) elog(ERROR, "invalid inet bit length: %d", bits); - /* clone the original data */ - dst = (inet *) palloc(VARSIZE_ANY(src)); - memcpy(dst, src, VARSIZE_ANY(src)); - - /* zero out any bits to the right of the netmask */ - byte = bits / 8; - - nbits = bits % 8; - /* clear the first byte, this might be a partial byte */ - if (nbits != 0) - { - ip_addr(dst)[byte] &= ~(0xFF >> nbits); - byte++; - } - /* clear remaining bytes */ - maxbytes = ip_addrsize(dst); - while (byte < maxbytes) - { - ip_addr(dst)[byte] = 0; - byte++; - } - - PG_RETURN_INET_P(dst); + PG_RETURN_INET_P(cidr_set_masklen_internal(src, bits)); } Datum @@ -334,10 +308,6 @@ cidr_set_masklen(PG_FUNCTION_ARGS) { inet *src = PG_GETARG_INET_PP(0); int bits = PG_GETARG_INT32(1); - inet *dst; - int byte; - int nbits; - int maxbytes; if (bits == -1) bits = ip_maxbits(src); @@ -347,31 +317,36 @@ cidr_set_masklen(PG_FUNCTION_ARGS) (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid mask length: %d", bits))); - /* clone the original data */ - dst = (inet *) palloc(VARSIZE_ANY(src)); - memcpy(dst, src, VARSIZE_ANY(src)); + PG_RETURN_INET_P(cidr_set_masklen_internal(src, bits)); +} - ip_bits(dst) = bits; +/* + * Copy src and set mask length to 'bits' (which must be valid for the family) + */ +inet * +cidr_set_masklen_internal(const inet *src, int bits) +{ + inet *dst = (inet *) palloc0(sizeof(inet)); - /* zero out any bits to the right of the new netmask */ - byte = bits / 8; + ip_family(dst) = ip_family(src); + ip_bits(dst) = bits; - nbits = bits % 8; - /* clear the first byte, this might be a partial byte */ - if (nbits != 0) + if (bits > 0) { - ip_addr(dst)[byte] &= ~(0xFF >> nbits); - byte++; - } - /* clear remaining bytes */ - maxbytes = ip_addrsize(dst); - while (byte < maxbytes) - { - ip_addr(dst)[byte] = 0; - byte++; + Assert(bits <= ip_maxbits(dst)); + + /* Clone appropriate bytes of the address, leaving the rest 0 */ + memcpy(ip_addr(dst), ip_addr(src), (bits + 7) / 8); + + /* Clear any unwanted bits in the last partial byte */ + if (bits % 8) + ip_addr(dst)[bits / 8] &= ~(0xFF >> (bits % 8)); } - PG_RETURN_INET_P(dst); + /* Set varlena header correctly */ + SET_INET_VARSIZE(dst); + + return dst; } /* @@ -719,11 +694,7 @@ network_broadcast(PG_FUNCTION_ARGS) /* make sure any unused bits are zeroed */ dst = (inet *) palloc0(sizeof(inet)); - if (ip_family(ip) == PGSQL_AF_INET) - maxbytes = 4; - else - maxbytes = 16; - + maxbytes = ip_addrsize(ip); bits = ip_bits(ip); a = ip_addr(ip); b = ip_addr(dst); @@ -853,11 +824,7 @@ network_hostmask(PG_FUNCTION_ARGS) /* make sure any unused bits are zeroed */ dst = (inet *) palloc0(sizeof(inet)); - if (ip_family(ip) == PGSQL_AF_INET) - maxbytes = 4; - else - maxbytes = 16; - + maxbytes = ip_addrsize(ip); bits = ip_maxbits(ip) - ip_bits(ip); b = ip_addr(dst); @@ -907,8 +874,7 @@ Datum inet_merge(PG_FUNCTION_ARGS) { inet *a1 = PG_GETARG_INET_PP(0), - *a2 = PG_GETARG_INET_PP(1), - *result; + *a2 = PG_GETARG_INET_PP(1); int commonbits; if (ip_family(a1) != ip_family(a2)) @@ -919,24 +885,7 @@ inet_merge(PG_FUNCTION_ARGS) commonbits = bitncommon(ip_addr(a1), ip_addr(a2), Min(ip_bits(a1), ip_bits(a2))); - /* Make sure any unused bits are zeroed. */ - result = (inet *) palloc0(sizeof(inet)); - - ip_family(result) = ip_family(a1); - ip_bits(result) = commonbits; - - /* Clone appropriate bytes of the address. */ - if (commonbits > 0) - memcpy(ip_addr(result), ip_addr(a1), (commonbits + 7) / 8); - - /* Clean any unwanted bits in the last partial byte. */ - if (commonbits % 8 != 0) - ip_addr(result)[commonbits / 8] &= ~(0xFF >> (commonbits % 8)); - - /* Set varlena header correctly. */ - SET_INET_VARSIZE(result); - - PG_RETURN_INET_P(result); + PG_RETURN_INET_P(cidr_set_masklen_internal(a1, commonbits)); } /* diff --git a/src/include/utils/inet.h b/src/include/utils/inet.h index 2fe3ca8c3c..dfa0b9f711 100644 --- a/src/include/utils/inet.h +++ b/src/include/utils/inet.h @@ -28,10 +28,12 @@ typedef struct } inet_struct; /* + * We use these values for the "family" field. + * * Referencing all of the non-AF_INET types to AF_INET lets us work on * machines which may not have the appropriate address family (like * inet6 addresses when AF_INET6 isn't present) but doesn't cause a - * dump/reload requirement. Existing databases used AF_INET for the family + * dump/reload requirement. Pre-7.4 databases used AF_INET for the family * type on disk. */ #define PGSQL_AF_INET (AF_INET + 0) @@ -117,6 +119,7 @@ typedef struct macaddr /* * Support functions in network.c */ +extern inet *cidr_set_masklen_internal(const inet *src, int bits); extern int bitncmp(const unsigned char *l, const unsigned char *r, int n); extern int bitncommon(const unsigned char *l, const unsigned char *r, int n); From ff36700c3ba2180047b4103de440ffaa34889b72 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 23 Aug 2016 10:05:13 -0400 Subject: [PATCH 052/871] Remove duplicate word from comment. Erik Rijkers --- src/backend/storage/lmgr/lmgr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c index eeedc38251..cbee20e9bf 100644 --- a/src/backend/storage/lmgr/lmgr.c +++ b/src/backend/storage/lmgr/lmgr.c @@ -33,7 +33,7 @@ * constraint violations. It's theoretically possible that a backend sees a * tuple that was speculatively inserted by another backend, but before it has * started waiting on the token, the other backend completes its insertion, - * and then then performs 2^32 unrelated insertions. And after all that, the + * and then performs 2^32 unrelated insertions. And after all that, the * first backend finally calls SpeculativeInsertionLockAcquire(), with the * intention of waiting for the first insertion to complete, but ends up * waiting for the latest unrelated insertion instead. Even then, nothing From 86f31695f3b54211226949de519063bbf248e8c4 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 23 Aug 2016 10:30:52 -0400 Subject: [PATCH 053/871] Add txid_current_ifassigned(). Add a variant of txid_current() that returns NULL if no transaction ID is assigned. This version can be used even on a standby server, although it will always return NULL since no transaction IDs can be assigned during recovery. Craig Ringer, per suggestion from Jim Nasby. Reviewed by Petr Jelinek and by me. --- doc/src/sgml/func.sgml | 9 +++++++++ src/backend/utils/adt/txid.c | 21 +++++++++++++++++++++ src/include/catalog/pg_proc.h | 2 ++ src/include/utils/builtins.h | 1 + src/test/regress/expected/txid.out | 16 ++++++++++++++++ src/test/regress/sql/txid.sql | 7 +++++++ 6 files changed, 56 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 169a385a9c..6355300d9d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -17119,6 +17119,10 @@ SELECT collation for ('foo' COLLATE "de_DE"); txid_current + + txid_current_if_assigned + + txid_current_snapshot @@ -17159,6 +17163,11 @@ SELECT collation for ('foo' COLLATE "de_DE"); bigint get current transaction ID, assigning a new one if the current transaction does not have one + + txid_current_if_assigned() + bigint + same as txid_current() but returns null instead of assigning an xid if none is already assigned + txid_current_snapshot() txid_snapshot diff --git a/src/backend/utils/adt/txid.c b/src/backend/utils/adt/txid.c index c2069a9923..276075e293 100644 --- a/src/backend/utils/adt/txid.c +++ b/src/backend/utils/adt/txid.c @@ -376,6 +376,27 @@ txid_current(PG_FUNCTION_ARGS) PG_RETURN_INT64(val); } +/* + * Same as txid_current() but doesn't assign a new xid if there isn't one + * yet. + */ +Datum +txid_current_if_assigned(PG_FUNCTION_ARGS) +{ + txid val; + TxidEpoch state; + TransactionId topxid = GetTopTransactionIdIfAny(); + + if (topxid == InvalidTransactionId) + PG_RETURN_NULL(); + + load_xid_epoch(&state); + + val = convert_xid(topxid, &state); + + PG_RETURN_INT64(val); +} + /* * txid_current_snapshot() returns txid_snapshot * diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 6fed7a0d19..050a98c397 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -4904,6 +4904,8 @@ DATA(insert OID = 2942 ( txid_snapshot_send PGNSP PGUID 12 1 0 0 0 f f f f t DESCR("I/O"); DATA(insert OID = 2943 ( txid_current PGNSP PGUID 12 1 0 0 0 f f f f t f s u 0 0 20 "" _null_ _null_ _null_ _null_ _null_ txid_current _null_ _null_ _null_ )); DESCR("get current transaction ID"); +DATA(insert OID = 3348 ( txid_current_if_assigned PGNSP PGUID 12 1 0 0 0 f f f f t f s u 0 0 20 "" _null_ _null_ _null_ _null_ _null_ txid_current_if_assigned _null_ _null_ _null_ )); +DESCR("get current transaction ID"); DATA(insert OID = 2944 ( txid_current_snapshot PGNSP PGUID 12 1 0 0 0 f f f f t f s s 0 0 2970 "" _null_ _null_ _null_ _null_ _null_ txid_current_snapshot _null_ _null_ _null_ )); DESCR("get current snapshot"); DATA(insert OID = 2945 ( txid_snapshot_xmin PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 20 "2970" _null_ _null_ _null_ _null_ _null_ txid_snapshot_xmin _null_ _null_ _null_ )); diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 40e25c8824..2ae212a9c3 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -1221,6 +1221,7 @@ extern Datum txid_snapshot_out(PG_FUNCTION_ARGS); extern Datum txid_snapshot_recv(PG_FUNCTION_ARGS); extern Datum txid_snapshot_send(PG_FUNCTION_ARGS); extern Datum txid_current(PG_FUNCTION_ARGS); +extern Datum txid_current_if_assigned(PG_FUNCTION_ARGS); extern Datum txid_current_snapshot(PG_FUNCTION_ARGS); extern Datum txid_snapshot_xmin(PG_FUNCTION_ARGS); extern Datum txid_snapshot_xmax(PG_FUNCTION_ARGS); diff --git a/src/test/regress/expected/txid.out b/src/test/regress/expected/txid.out index ddd217eb10..802ccb949f 100644 --- a/src/test/regress/expected/txid.out +++ b/src/test/regress/expected/txid.out @@ -238,3 +238,19 @@ SELECT txid_snapshot '1:9223372036854775808:3'; ERROR: invalid input syntax for type txid_snapshot: "1:9223372036854775808:3" LINE 1: SELECT txid_snapshot '1:9223372036854775808:3'; ^ +-- test txid_current_if_assigned +BEGIN; +SELECT txid_current_if_assigned() IS NULL; + ?column? +---------- + t +(1 row) + +SELECT txid_current() \gset +SELECT txid_current_if_assigned() IS NOT DISTINCT FROM BIGINT :'txid_current'; + ?column? +---------- + t +(1 row) + +COMMIT; diff --git a/src/test/regress/sql/txid.sql b/src/test/regress/sql/txid.sql index b6650b922e..4aefd9e64d 100644 --- a/src/test/regress/sql/txid.sql +++ b/src/test/regress/sql/txid.sql @@ -52,3 +52,10 @@ select txid_visible_in_snapshot('1000100010001015', '1000100010001000:1000100010 -- test 64bit overflow SELECT txid_snapshot '1:9223372036854775807:3'; SELECT txid_snapshot '1:9223372036854775808:3'; + +-- test txid_current_if_assigned +BEGIN; +SELECT txid_current_if_assigned() IS NULL; +SELECT txid_current() \gset +SELECT txid_current_if_assigned() IS NOT DISTINCT FROM BIGINT :'txid_current'; +COMMIT; From d2ddee63b43b27d6c6af169342af10db19bd3a1a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 23 Aug 2016 12:10:25 -0400 Subject: [PATCH 054/871] Improve SP-GiST opclass API to better support unlabeled nodes. Previously, the spgSplitTuple action could only create a new upper tuple containing a single labeled node. This made it useless for opclasses that prefer to work with fixed sets of nodes (labeled or otherwise), which meant that restrictive prefixes could not be used with such node definitions. Change the output field set for the choose() method to allow it to specify any valid node set for the new upper tuple, and to specify which of these nodes to place the modified lower tuple in. In addition to its primary use for fixed node sets, this feature could allow existing opclasses that use variable node sets to skip a separate spgAddNode action when splitting a tuple, by setting up the node needed for the incoming value as part of the spgSplitTuple action. However, care would have to be taken to add the extra node only when it would not make the tuple bigger than before. (spgAddNode can enlarge the tuple, spgSplitTuple can't.) This is a prerequisite for an upcoming SP-GiST inet opclass, but is being committed separately to increase the visibility of the API change. In passing, improve the documentation about the traverse-values feature that was added by commit ccd6eb49a. Emre Hasegeli, with cosmetic adjustments and documentation rework by me Discussion: --- doc/src/sgml/spgist.sgml | 115 +++++++++++++----------- src/backend/access/spgist/spgdoinsert.c | 39 ++++++-- src/backend/access/spgist/spgtextproc.c | 12 ++- src/include/access/spgist.h | 12 ++- 4 files changed, 115 insertions(+), 63 deletions(-) diff --git a/doc/src/sgml/spgist.sgml b/doc/src/sgml/spgist.sgml index f40c790612..dfa62adb55 100644 --- a/doc/src/sgml/spgist.sgml +++ b/doc/src/sgml/spgist.sgml @@ -114,7 +114,7 @@ box_ops - box + box << &< @@ -183,11 +183,14 @@ Inner tuples are more complex, since they are branching points in the search tree. Each inner tuple contains a set of one or more nodes, which represent groups of similar leaf values. - A node contains a downlink that leads to either another, lower-level inner - tuple, or a short list of leaf tuples that all lie on the same index page. - Each node has a label that describes it; for example, + A node contains a downlink that leads either to another, lower-level inner + tuple, or to a short list of leaf tuples that all lie on the same index page. + Each node normally has a label that describes it; for example, in a radix tree the node label could be the next character of the string - value. Optionally, an inner tuple can have a prefix value + value. (Alternatively, an operator class can omit the node labels, if it + works with a fixed set of nodes for all inner tuples; + see .) + Optionally, an inner tuple can have a prefix value that describes all its members. In a radix tree this could be the common prefix of the represented strings. The prefix value is not necessarily really a prefix, but can be any data needed by the operator class; @@ -202,7 +205,8 @@ tuple, so the SP-GiST core provides the possibility for operator classes to manage level counting while descending the tree. There is also support for incrementally reconstructing the represented - value when that is needed. + value when that is needed, and for passing down additional data (called + traverse values) during a tree descent. @@ -343,10 +347,13 @@ typedef struct spgChooseOut } addNode; struct /* results for spgSplitTuple */ { - /* Info to form new inner tuple with one node */ + /* Info to form new upper-level inner tuple with one child tuple */ bool prefixHasPrefix; /* tuple should have a prefix? */ Datum prefixPrefixDatum; /* if so, its value */ - Datum nodeLabel; /* node's label */ + int prefixNNodes; /* number of nodes */ + Datum *prefixNodeLabels; /* their labels (or NULL for + * no labels) */ + int childNodeN; /* which node gets child tuple */ /* Info to form new lower-level inner tuple with all old nodes */ bool postfixHasPrefix; /* tuple should have a prefix? */ @@ -416,29 +423,33 @@ typedef struct spgChooseOut set resultType to spgSplitTuple. This action moves all the existing nodes into a new lower-level inner tuple, and replaces the existing inner tuple with a tuple - having a single node that links to the new lower-level inner tuple. + having a single downlink pointing to the new lower-level inner tuple. Set prefixHasPrefix to indicate whether the new upper tuple should have a prefix, and if so set prefixPrefixDatum to the prefix value. This new prefix value must be sufficiently less restrictive than the original - to accept the new value to be indexed, and it should be no longer - than the original prefix. - Set nodeLabel to the label to be used for the - node that will point to the new lower-level inner tuple. + to accept the new value to be indexed. + Set prefixNNodes to the number of nodes needed in the + new tuple, and set prefixNodeLabels to a palloc'd array + holding their labels, or to NULL if node labels are not required. + Note that the total size of the new upper tuple must be no more + than the total size of the tuple it is replacing; this constrains + the lengths of the new prefix and new labels. + Set childNodeN to the index (from zero) of the node + that will downlink to the new lower-level inner tuple. Set postfixHasPrefix to indicate whether the new lower-level inner tuple should have a prefix, and if so set postfixPrefixDatum to the prefix value. The - combination of these two prefixes and the additional label must - have the same meaning as the original prefix, because there is - no opportunity to alter the node labels that are moved to the new - lower-level tuple, nor to change any child index entries. + combination of these two prefixes and the downlink node's label + (if any) must have the same meaning as the original prefix, because + there is no opportunity to alter the node labels that are moved to + the new lower-level tuple, nor to change any child index entries. After the node has been split, the choose function will be called again with the replacement inner tuple. - That call will usually result in an spgAddNode result, - since presumably the node label added in the split step will not - match the new value; so after that, there will be a third call - that finally returns spgMatchNode and allows the - insertion to descend to the leaf level. + That call may return an spgAddNode result, if no suitable + node was created by the spgSplitTuple action. Eventually + choose must return spgMatchNode to + allow the insertion to descend to the next level. @@ -492,9 +503,8 @@ typedef struct spgPickSplitOut prefixDatum to the prefix value. Set nNodes to indicate the number of nodes that the new inner tuple will contain, and - set nodeLabels to an array of their label values. - (If the nodes do not require labels, set nodeLabels - to NULL; see for details.) + set nodeLabels to an array of their label values, + or to NULL if node labels are not required. Set mapTuplesToNodes to an array that gives the index (from zero) of the node that each leaf tuple should be assigned to. Set leafTupleDatums to an array of the values to @@ -561,7 +571,7 @@ typedef struct spgInnerConsistentIn Datum reconstructedValue; /* value reconstructed at parent */ void *traversalValue; /* opclass-specific traverse value */ - MemoryContext traversalMemoryContext; + MemoryContext traversalMemoryContext; /* put new traverse values here */ int level; /* current level (counting from zero) */ bool returnData; /* original data must be returned? */ @@ -580,7 +590,6 @@ typedef struct spgInnerConsistentOut int *levelAdds; /* increment level by this much for each */ Datum *reconstructedValues; /* associated reconstructed values */ void **traversalValues; /* opclass-specific traverse values */ - } spgInnerConsistentOut; @@ -599,6 +608,11 @@ typedef struct spgInnerConsistentOut parent tuple; it is (Datum) 0 at the root level or if the inner_consistent function did not provide a value at the parent level. + traversalValue is a pointer to any traverse data + passed down from the previous call of inner_consistent + on the parent index tuple, or NULL at the root level. + traversalMemoryContext is the memory context in which + to store output traverse values (see below). level is the current inner tuple's level, starting at zero for the root level. returnData is true if reconstructed data is @@ -615,9 +629,6 @@ typedef struct spgInnerConsistentOut inner tuple, and nodeLabels is an array of their label values, or NULL if the nodes do not have labels. - traversalValue is a pointer to data that - inner_consistent gets when called on child nodes from an - outer call of inner_consistent on parent nodes. @@ -633,17 +644,19 @@ typedef struct spgInnerConsistentOut reconstructedValues to an array of the values reconstructed for each child node to be visited; otherwise, leave reconstructedValues as NULL. + If it is desired to pass down additional out-of-band information + (traverse values) to lower levels of the tree search, + set traversalValues to an array of the appropriate + traverse values, one for each child node to be visited; otherwise, + leave traversalValues as NULL. Note that the inner_consistent function is responsible for palloc'ing the - nodeNumbers, levelAdds and - reconstructedValues arrays. - Sometimes accumulating some information is needed, while - descending from parent to child node was happened. In this case - traversalValues array keeps pointers to - specific data you need to accumulate for every child node. - Memory for traversalValues should be allocated in - the default context, but each element of it should be allocated in - traversalMemoryContext. + nodeNumbers, levelAdds, + reconstructedValues, and + traversalValues arrays in the current memory context. + However, any output traverse values pointed to by + the traversalValues array should be allocated + in traversalMemoryContext. @@ -670,8 +683,8 @@ typedef struct spgLeafConsistentIn ScanKey scankeys; /* array of operators and comparison values */ int nkeys; /* length of array */ - void *traversalValue; /* opclass-specific traverse value */ Datum reconstructedValue; /* value reconstructed at parent */ + void *traversalValue; /* opclass-specific traverse value */ int level; /* current level (counting from zero) */ bool returnData; /* original data must be returned? */ @@ -700,6 +713,9 @@ typedef struct spgLeafConsistentOut parent tuple; it is (Datum) 0 at the root level or if the inner_consistent function did not provide a value at the parent level. + traversalValue is a pointer to any traverse data + passed down from the previous call of inner_consistent + on the parent index tuple, or NULL at the root level. level is the current leaf tuple's level, starting at zero for the root level. returnData is true if reconstructed data is @@ -797,7 +813,10 @@ typedef struct spgLeafConsistentOut point. In such a case the code typically works with the nodes by number, and there is no need for explicit node labels. To suppress node labels (and thereby save some space), the picksplit - function can return NULL for the nodeLabels array. + function can return NULL for the nodeLabels array, + and likewise the choose function can return NULL for + the prefixNodeLabels array during + a spgSplitTuple action. This will in turn result in nodeLabels being NULL during subsequent calls to choose and inner_consistent. In principle, node labels could be used for some inner tuples and omitted @@ -807,10 +826,7 @@ typedef struct spgLeafConsistentOut When working with an inner tuple having unlabeled nodes, it is an error for choose to return spgAddNode, since the set - of nodes is supposed to be fixed in such cases. Also, there is no - provision for generating an unlabeled node in spgSplitTuple - actions, since it is expected that an spgAddNode action will - be needed as well. + of nodes is supposed to be fixed in such cases. @@ -859,11 +875,10 @@ typedef struct spgLeafConsistentOut The PostgreSQL source distribution includes - several examples of index operator classes for - SP-GiST. The core system currently provides radix - trees over text columns and two types of trees over points: quad-tree and - k-d tree. Look into src/backend/access/spgist/ to see the - code. + several examples of index operator classes for SP-GiST, + as described in . Look + into src/backend/access/spgist/ + and src/backend/utils/adt/ to see the code. diff --git a/src/backend/access/spgist/spgdoinsert.c b/src/backend/access/spgist/spgdoinsert.c index f090ca528b..6fc04b224d 100644 --- a/src/backend/access/spgist/spgdoinsert.c +++ b/src/backend/access/spgist/spgdoinsert.c @@ -1705,17 +1705,40 @@ spgSplitNodeAction(Relation index, SpGistState *state, /* Should not be applied to nulls */ Assert(!SpGistPageStoresNulls(current->page)); + /* Check opclass gave us sane values */ + if (out->result.splitTuple.prefixNNodes <= 0 || + out->result.splitTuple.prefixNNodes > SGITMAXNNODES) + elog(ERROR, "invalid number of prefix nodes: %d", + out->result.splitTuple.prefixNNodes); + if (out->result.splitTuple.childNodeN < 0 || + out->result.splitTuple.childNodeN >= + out->result.splitTuple.prefixNNodes) + elog(ERROR, "invalid child node number: %d", + out->result.splitTuple.childNodeN); + /* - * Construct new prefix tuple, containing a single node with the specified - * label. (We'll update the node's downlink to point to the new postfix - * tuple, below.) + * Construct new prefix tuple with requested number of nodes. We'll fill + * in the childNodeN'th node's downlink below. */ - node = spgFormNodeTuple(state, out->result.splitTuple.nodeLabel, false); + nodes = (SpGistNodeTuple *) palloc(sizeof(SpGistNodeTuple) * + out->result.splitTuple.prefixNNodes); + + for (i = 0; i < out->result.splitTuple.prefixNNodes; i++) + { + Datum label = (Datum) 0; + bool labelisnull; + + labelisnull = (out->result.splitTuple.prefixNodeLabels == NULL); + if (!labelisnull) + label = out->result.splitTuple.prefixNodeLabels[i]; + nodes[i] = spgFormNodeTuple(state, label, labelisnull); + } prefixTuple = spgFormInnerTuple(state, out->result.splitTuple.prefixHasPrefix, out->result.splitTuple.prefixPrefixDatum, - 1, &node); + out->result.splitTuple.prefixNNodes, + nodes); /* it must fit in the space that innerTuple now occupies */ if (prefixTuple->size > innerTuple->size) @@ -1807,10 +1830,12 @@ spgSplitNodeAction(Relation index, SpGistState *state, * the postfix tuple first.) We have to update the local copy of the * prefixTuple too, because that's what will be written to WAL. */ - spgUpdateNodeLink(prefixTuple, 0, postfixBlkno, postfixOffset); + spgUpdateNodeLink(prefixTuple, out->result.splitTuple.childNodeN, + postfixBlkno, postfixOffset); prefixTuple = (SpGistInnerTuple) PageGetItem(current->page, PageGetItemId(current->page, current->offnum)); - spgUpdateNodeLink(prefixTuple, 0, postfixBlkno, postfixOffset); + spgUpdateNodeLink(prefixTuple, out->result.splitTuple.childNodeN, + postfixBlkno, postfixOffset); MarkBufferDirty(current->buffer); diff --git a/src/backend/access/spgist/spgtextproc.c b/src/backend/access/spgist/spgtextproc.c index e0d8f30ef1..852a9b00fa 100644 --- a/src/backend/access/spgist/spgtextproc.c +++ b/src/backend/access/spgist/spgtextproc.c @@ -212,9 +212,14 @@ spg_text_choose(PG_FUNCTION_ARGS) out->result.splitTuple.prefixPrefixDatum = formTextDatum(prefixStr, commonLen); } - out->result.splitTuple.nodeLabel = + out->result.splitTuple.prefixNNodes = 1; + out->result.splitTuple.prefixNodeLabels = + (Datum *) palloc(sizeof(Datum)); + out->result.splitTuple.prefixNodeLabels[0] = Int16GetDatum(*(unsigned char *) (prefixStr + commonLen)); + out->result.splitTuple.childNodeN = 0; + if (prefixSize - commonLen == 1) { out->result.splitTuple.postfixHasPrefix = false; @@ -280,7 +285,10 @@ spg_text_choose(PG_FUNCTION_ARGS) out->resultType = spgSplitTuple; out->result.splitTuple.prefixHasPrefix = in->hasPrefix; out->result.splitTuple.prefixPrefixDatum = in->prefixDatum; - out->result.splitTuple.nodeLabel = Int16GetDatum(-2); + out->result.splitTuple.prefixNNodes = 1; + out->result.splitTuple.prefixNodeLabels = (Datum *) palloc(sizeof(Datum)); + out->result.splitTuple.prefixNodeLabels[0] = Int16GetDatum(-2); + out->result.splitTuple.childNodeN = 0; out->result.splitTuple.postfixHasPrefix = false; } else diff --git a/src/include/access/spgist.h b/src/include/access/spgist.h index f39a2d6938..a953a5a401 100644 --- a/src/include/access/spgist.h +++ b/src/include/access/spgist.h @@ -90,10 +90,13 @@ typedef struct spgChooseOut } addNode; struct /* results for spgSplitTuple */ { - /* Info to form new inner tuple with one node */ + /* Info to form new upper-level inner tuple with one child tuple */ bool prefixHasPrefix; /* tuple should have a prefix? */ Datum prefixPrefixDatum; /* if so, its value */ - Datum nodeLabel; /* node's label */ + int prefixNNodes; /* number of nodes */ + Datum *prefixNodeLabels; /* their labels (or NULL for + * no labels) */ + int childNodeN; /* which node gets child tuple */ /* Info to form new lower-level inner tuple with all old nodes */ bool postfixHasPrefix; /* tuple should have a prefix? */ @@ -134,7 +137,8 @@ typedef struct spgInnerConsistentIn Datum reconstructedValue; /* value reconstructed at parent */ void *traversalValue; /* opclass-specific traverse value */ - MemoryContext traversalMemoryContext; + MemoryContext traversalMemoryContext; /* put new traverse values + * here */ int level; /* current level (counting from zero) */ bool returnData; /* original data must be returned? */ @@ -163,8 +167,8 @@ typedef struct spgLeafConsistentIn ScanKey scankeys; /* array of operators and comparison values */ int nkeys; /* length of array */ - void *traversalValue; /* opclass-specific traverse value */ Datum reconstructedValue; /* value reconstructed at parent */ + void *traversalValue; /* opclass-specific traverse value */ int level; /* current level (counting from zero) */ bool returnData; /* original data must be returned? */ From ff066481b0485b1a4e414de3abcaae0bda02b1e1 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 23 Aug 2016 12:45:33 -0400 Subject: [PATCH 055/871] doc: fix incorrect 'literal' tags Discussion: dcc4113d-1eda-4f60-d1c5-f50eee160bad@gmail.com Author: Alexander Law Backpatch-through: 9.6 --- doc/src/sgml/pgstandby.sgml | 2 +- doc/src/sgml/ref/pg_xlogdump.sgml | 2 +- doc/src/sgml/ref/pgbench.sgml | 2 +- doc/src/sgml/ref/psql-ref.sgml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/pgstandby.sgml b/doc/src/sgml/pgstandby.sgml index fb3f32eaaa..80c6f60062 100644 --- a/doc/src/sgml/pgstandby.sgml +++ b/doc/src/sgml/pgstandby.sgml @@ -363,7 +363,7 @@ recovery_end_command = 'del C:\pgsql.trigger.5442' The copy command on Windows sets the final file size before the file is completely copied, which would ordinarily confuse pg_standby. Therefore - pg_standby waits sleeptime + pg_standby waits sleeptime seconds once it sees the proper file size. GNUWin32's cp sets the file size only after the file copy is complete. diff --git a/doc/src/sgml/ref/pg_xlogdump.sgml b/doc/src/sgml/ref/pg_xlogdump.sgml index 296f1acc24..177caab00d 100644 --- a/doc/src/sgml/ref/pg_xlogdump.sgml +++ b/doc/src/sgml/ref/pg_xlogdump.sgml @@ -153,7 +153,7 @@ PostgreSQL documentation Timeline from which to read log records. The default is to use the - value in startseg, if that is specified; otherwise, the + value in startseg, if that is specified; otherwise, the default is 1. diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml index f58da351f9..285608d508 100644 --- a/doc/src/sgml/ref/pgbench.sgml +++ b/doc/src/sgml/ref/pgbench.sgml @@ -433,7 +433,7 @@ pgbench options dbname sec - Show progress report every sec seconds. The report + Show progress report every sec seconds. The report includes the time since the beginning of the run, the tps since the last report, and the transaction latency average and standard deviation since the last report. Under throttling ( - If this is on, you should create users as username@dbname. - When username is passed by a connecting client, + If this is on, you should create users as username@dbname. + When username is passed by a connecting client, @ and the database name are appended to the user name and that database-specific user name is looked up by the server. Note that when you create users with names containing diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 6355300d9d..5c1c4f69fb 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -13800,7 +13800,7 @@ SELECT xmlagg(x) FROM (SELECT x FROM test ORDER BY y DESC) AS tab; No multiple continuous percentile: returns an array of results matching - the shape of the fractions parameter, with each + the shape of the fractions parameter, with each non-null element replaced by the value corresponding to that percentile @@ -13845,7 +13845,7 @@ SELECT xmlagg(x) FROM (SELECT x FROM test ORDER BY y DESC) AS tab; No multiple discrete percentile: returns an array of results matching the - shape of the fractions parameter, with each non-null + shape of the fractions parameter, with each non-null element replaced by the input value corresponding to that percentile @@ -16850,7 +16850,7 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); pg_options_to_table returns the set of storage option name/value pairs - (option_name/option_value) when passed + (option_name/option_value) when passed pg_class.reloptions or pg_attribute.attoptions. diff --git a/doc/src/sgml/ref/create_sequence.sgml b/doc/src/sgml/ref/create_sequence.sgml index c9591462ee..62ae379226 100644 --- a/doc/src/sgml/ref/create_sequence.sgml +++ b/doc/src/sgml/ref/create_sequence.sgml @@ -349,7 +349,7 @@ END; - The standard's AS <data type> expression is not + The standard's AS data_type expression is not supported. diff --git a/doc/src/sgml/ref/set_role.sgml b/doc/src/sgml/ref/set_role.sgml index aff3792199..a97ceabcff 100644 --- a/doc/src/sgml/ref/set_role.sgml +++ b/doc/src/sgml/ref/set_role.sgml @@ -127,7 +127,7 @@ SELECT SESSION_USER, CURRENT_USER; PostgreSQL - allows identifier syntax ("rolename"), while + allows identifier syntax ("rolename"), while the SQL standard requires the role name to be written as a string literal. SQL does not allow this command during a transaction; PostgreSQL does not make this diff --git a/doc/src/sgml/ref/set_session_auth.sgml b/doc/src/sgml/ref/set_session_auth.sgml index 4ac2128950..96d279aaf9 100644 --- a/doc/src/sgml/ref/set_session_auth.sgml +++ b/doc/src/sgml/ref/set_session_auth.sgml @@ -101,7 +101,7 @@ SELECT SESSION_USER, CURRENT_USER; The SQL standard allows some other expressions to appear in place of the literal user_name, but these options are not important in practice. PostgreSQL - allows identifier syntax ("username"), which SQL + allows identifier syntax ("username"), which SQL does not. SQL does not allow this command during a transaction; PostgreSQL does not make this restriction because there is no reason to. diff --git a/doc/src/sgml/textsearch.sgml b/doc/src/sgml/textsearch.sgml index be5974a4ff..5a70d7db80 100644 --- a/doc/src/sgml/textsearch.sgml +++ b/doc/src/sgml/textsearch.sgml @@ -3622,10 +3622,10 @@ SELECT plainto_tsquery('supernovae stars'); - The optional parameter PATTERN can be the name of + The optional parameter PATTERN can be the name of a text search object, optionally schema-qualified. If - PATTERN is omitted then information about all - visible objects will be displayed. PATTERN can be a + PATTERN is omitted then information about all + visible objects will be displayed. PATTERN can be a regular expression and can provide separate patterns for the schema and object names. The following examples illustrate this: diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index d8d2e9e490..de6a466efc 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -204,8 +204,8 @@ SELECT clean_emp(); If an argument is of a composite type, then the dot notation, - e.g., argname.fieldname or - $1.fieldname, can be used to access attributes of the + e.g., argname.fieldname or + $1.fieldname, can be used to access attributes of the argument. Again, you might need to qualify the argument's name with the function name to make the form with an argument name unambiguous. @@ -527,7 +527,8 @@ LINE 1: SELECT new_emp().name; Another option is to use functional notation for extracting an attribute. The simple way to explain this is that we can use the - notations attribute(table) and table.attribute + notations attribute(table) + and table.attribute interchangeably. @@ -1305,12 +1306,15 @@ CREATE FUNCTION test(smallint, double precision) RETURNS ... A function that takes a single argument of a composite type should generally not have the same name as any attribute (field) of that type. - Recall that attribute(table) is considered equivalent - to table.attribute. In the case that there is an + Recall that attribute(table) + is considered equivalent + to table.attribute. + In the case that there is an ambiguity between a function on a composite type and an attribute of the composite type, the attribute will always be used. It is possible to override that choice by schema-qualifying the function name - (that is, schema.func(table)) but it's better to + (that is, schema.func(table) + ) but it's better to avoid the problem by not choosing conflicting names. @@ -2818,7 +2822,7 @@ HeapTuple heap_form_tuple(TupleDesc tupdesc, Datum *values, bool *isnull) HeapTuple BuildTupleFromCStrings(AttInMetadata *attinmeta, char **values) to build a HeapTuple given user data - in C string form. values is an array of C strings, + in C string form. values is an array of C strings, one for each attribute of the return row. Each C string should be in the form expected by the input function of the attribute data type. In order to return a null value for one of the attributes, From ae4760d667c71924932ab32e14996b5be1831fc6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 24 Aug 2016 22:20:01 -0400 Subject: [PATCH 068/871] Fix small query-lifespan memory leak in bulk updates. When there is an identifiable REPLICA IDENTITY index on the target table, heap_update leaks the id_attrs bitmapset. That's not many bytes, but it adds up over enough rows, since the code typically runs in a query-lifespan context. Bug introduced in commit e55704d8b, which did a rather poor job of cloning the existing use-pattern for RelationGetIndexAttrBitmap(). Per bug #14293 from Zhou Digoal. Back-patch to 9.4 where the bug was introduced. Report: <20160824114320.15676.45171@wrigleys.postgresql.org> --- src/backend/access/heap/heapam.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index c63dfa0baf..6a27ef4140 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -3802,6 +3802,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, ReleaseBuffer(vmbuffer); bms_free(hot_attrs); bms_free(key_attrs); + bms_free(id_attrs); return result; } @@ -4268,6 +4269,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, bms_free(hot_attrs); bms_free(key_attrs); + bms_free(id_attrs); return HeapTupleMayBeUpdated; } From 2533ff0aa518d4d31391db279cf08e538fae5931 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 25 Aug 2016 09:57:09 -0400 Subject: [PATCH 069/871] Fix instability in parallel regression tests. Commit f0c7b789a added a test case in case.sql that creates and then drops both an '=' operator and the type it's for. Given the right timing, that can cause a "cache lookup failed for type" failure in concurrent sessions, which see the '=' operator as a potential match for '=' in a query, but then the type is gone by the time they inquire into its properties. It might be nice to make that behavior more robust someday, but as a back-patchable solution, adjust the new test case so that the operator is never visible to other sessions. Like the previous commit, back-patch to all supported branches. Discussion: <5983.1471371667@sss.pgh.pa.us> --- src/test/regress/expected/case.out | 9 ++++----- src/test/regress/sql/case.sql | 11 ++++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/regress/expected/case.out b/src/test/regress/expected/case.out index 35b6476e50..5f6aa16d31 100644 --- a/src/test/regress/expected/case.out +++ b/src/test/regress/expected/case.out @@ -305,6 +305,9 @@ SELECT * FROM CASE_TBL; -- the isNull flag for the case test value incorrectly became true, causing -- the third WHEN-clause not to match. The volatile function calls are needed -- to prevent constant-folding in the planner, which would hide the bug. +-- Wrap this in a single transaction so the transient '=' operator doesn't +-- cause problems in concurrent sessions +BEGIN; CREATE FUNCTION vol(text) returns text as 'begin return $1; end' language plpgsql volatile; SELECT CASE @@ -335,13 +338,9 @@ SELECT CASE volfoo('bar') WHEN 'foo'::foodomain THEN 'is foo' ELSE 'is not foo' is not foo (1 row) +ROLLBACK; -- -- Clean up -- DROP TABLE CASE_TBL; DROP TABLE CASE2_TBL; -DROP OPERATOR = (foodomain, foodomain); -DROP FUNCTION inline_eq(foodomain, foodomain); -DROP FUNCTION volfoo(text); -DROP DOMAIN foodomain; -DROP FUNCTION vol(text); diff --git a/src/test/regress/sql/case.sql b/src/test/regress/sql/case.sql index b2377e4610..c860fae258 100644 --- a/src/test/regress/sql/case.sql +++ b/src/test/regress/sql/case.sql @@ -167,6 +167,10 @@ SELECT * FROM CASE_TBL; -- the third WHEN-clause not to match. The volatile function calls are needed -- to prevent constant-folding in the planner, which would hide the bug. +-- Wrap this in a single transaction so the transient '=' operator doesn't +-- cause problems in concurrent sessions +BEGIN; + CREATE FUNCTION vol(text) returns text as 'begin return $1; end' language plpgsql volatile; @@ -194,14 +198,11 @@ CREATE OPERATOR = (procedure = inline_eq, SELECT CASE volfoo('bar') WHEN 'foo'::foodomain THEN 'is foo' ELSE 'is not foo' END; +ROLLBACK; + -- -- Clean up -- DROP TABLE CASE_TBL; DROP TABLE CASE2_TBL; -DROP OPERATOR = (foodomain, foodomain); -DROP FUNCTION inline_eq(foodomain, foodomain); -DROP FUNCTION volfoo(text); -DROP DOMAIN foodomain; -DROP FUNCTION vol(text); From ae025a15988f5491903cd3a2075f308c2773f711 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 26 Aug 2016 16:33:57 +0300 Subject: [PATCH 070/871] Support OID system column in postgres_fdw. You can use ALTER FOREIGN TABLE SET WITH OIDS on a foreign table, but the oid column read out as zeros, because the postgres_fdw didn't know about it. Teach postgres_fdw how to fetch it. Etsuro Fujita, with an additional test case by me. Discussion: <56E90A76.5000503@lab.ntt.co.jp> --- contrib/postgres_fdw/deparse.c | 37 +++++++++++++--- .../postgres_fdw/expected/postgres_fdw.out | 43 ++++++++++++++----- contrib/postgres_fdw/postgres_fdw.c | 28 +++++++++++- contrib/postgres_fdw/sql/postgres_fdw.sql | 13 +++++- 4 files changed, 101 insertions(+), 20 deletions(-) diff --git a/contrib/postgres_fdw/deparse.c b/contrib/postgres_fdw/deparse.c index aaf9108c56..691658f099 100644 --- a/contrib/postgres_fdw/deparse.c +++ b/contrib/postgres_fdw/deparse.c @@ -287,13 +287,14 @@ foreign_expr_walker(Node *node, /* Var belongs to foreign table */ /* - * System columns other than ctid should not be sent to - * the remote, since we don't make any effort to ensure - * that local and remote values match (tableoid, in + * System columns other than ctid and oid should not be + * sent to the remote, since we don't make any effort to + * ensure that local and remote values match (tableoid, in * particular, almost certainly doesn't match). */ if (var->varattno < 0 && - var->varattno != SelfItemPointerAttributeNumber) + var->varattno != SelfItemPointerAttributeNumber && + var->varattno != ObjectIdAttributeNumber) return false; /* Else check the collation */ @@ -913,8 +914,8 @@ deparseTargetList(StringInfo buf, } /* - * Add ctid if needed. We currently don't support retrieving any other - * system columns. + * Add ctid and oid if needed. We currently don't support retrieving any + * other system columns. */ if (bms_is_member(SelfItemPointerAttributeNumber - FirstLowInvalidHeapAttributeNumber, attrs_used)) @@ -932,6 +933,22 @@ deparseTargetList(StringInfo buf, *retrieved_attrs = lappend_int(*retrieved_attrs, SelfItemPointerAttributeNumber); } + if (bms_is_member(ObjectIdAttributeNumber - FirstLowInvalidHeapAttributeNumber, + attrs_used)) + { + if (!first) + appendStringInfoString(buf, ", "); + else if (is_returning) + appendStringInfoString(buf, " RETURNING "); + first = false; + + if (qualify_col) + ADD_REL_QUALIFIER(buf, rtindex); + appendStringInfoString(buf, "oid"); + + *retrieved_attrs = lappend_int(*retrieved_attrs, + ObjectIdAttributeNumber); + } /* Don't generate bad syntax if no undropped columns */ if (first && !is_returning) @@ -1574,13 +1591,19 @@ deparseColumnRef(StringInfo buf, int varno, int varattno, PlannerInfo *root, { RangeTblEntry *rte; + /* We support fetching the remote side's CTID and OID. */ if (varattno == SelfItemPointerAttributeNumber) { - /* We support fetching the remote side's CTID. */ if (qualify_col) ADD_REL_QUALIFIER(buf, varno); appendStringInfoString(buf, "ctid"); } + else if (varattno == ObjectIdAttributeNumber) + { + if (qualify_col) + ADD_REL_QUALIFIER(buf, varno); + appendStringInfoString(buf, "oid"); + } else if (varattno < 0) { /* diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index d7747cc665..d97e694d1a 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -124,6 +124,13 @@ CREATE FOREIGN TABLE ft6 ( c2 int NOT NULL, c3 text ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4'); +-- A table with oids. CREATE FOREIGN TABLE doesn't support the +-- WITH OIDS option, but ALTER does. +CREATE FOREIGN TABLE ft_pg_type ( + typname name, + typlen smallint +) SERVER loopback OPTIONS (schema_name 'pg_catalog', table_name 'pg_type'); +ALTER TABLE ft_pg_type SET WITH OIDS; -- =================================================================== -- tests for validator -- =================================================================== @@ -173,15 +180,16 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1'); ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1'); ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1'); \det+ - List of foreign tables - Schema | Table | Server | FDW Options | Description ---------+-------+-----------+---------------------------------------+------------- - public | ft1 | loopback | (schema_name 'S 1', table_name 'T 1') | - public | ft2 | loopback | (schema_name 'S 1', table_name 'T 1') | - public | ft4 | loopback | (schema_name 'S 1', table_name 'T 3') | - public | ft5 | loopback | (schema_name 'S 1', table_name 'T 4') | - public | ft6 | loopback2 | (schema_name 'S 1', table_name 'T 4') | -(5 rows) + List of foreign tables + Schema | Table | Server | FDW Options | Description +--------+------------+-----------+--------------------------------------------------+------------- + public | ft1 | loopback | (schema_name 'S 1', table_name 'T 1') | + public | ft2 | loopback | (schema_name 'S 1', table_name 'T 1') | + public | ft4 | loopback | (schema_name 'S 1', table_name 'T 3') | + public | ft5 | loopback | (schema_name 'S 1', table_name 'T 4') | + public | ft6 | loopback2 | (schema_name 'S 1', table_name 'T 4') | + public | ft_pg_type | loopback | (schema_name 'pg_catalog', table_name 'pg_type') | +(6 rows) -- Now we should be able to run ANALYZE. -- To exercise multiple code paths, we use local stats on ft1 @@ -2485,7 +2493,7 @@ DEALLOCATE st2; DEALLOCATE st3; DEALLOCATE st4; DEALLOCATE st5; --- System columns, except ctid, should not be sent to remote +-- System columns, except ctid and oid, should not be sent to remote EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.tableoid = 'pg_class'::regclass LIMIT 1; QUERY PLAN @@ -2553,6 +2561,21 @@ SELECT ctid, * FROM ft1 t1 LIMIT 1; (0,1) | 1 | 1 | 00001 | Fri Jan 02 00:00:00 1970 PST | Fri Jan 02 00:00:00 1970 | 1 | 1 | foo (1 row) +EXPLAIN (VERBOSE, COSTS OFF) +SELECT oid, * FROM ft_pg_type WHERE typname = 'int4'; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Foreign Scan on public.ft_pg_type + Output: oid, typname, typlen + Remote SQL: SELECT typname, typlen, oid FROM pg_catalog.pg_type WHERE ((typname = 'int4'::name)) +(3 rows) + +SELECT oid, * FROM ft_pg_type WHERE typname = 'int4'; + oid | typname | typlen +-----+---------+-------- + 23 | int4 | 4 +(1 row) + -- =================================================================== -- used in pl/pgsql function -- =================================================================== diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index bfd81c46c8..b92f29958f 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -4374,6 +4374,7 @@ make_tuple_from_result_row(PGresult *res, Datum *values; bool *nulls; ItemPointer ctid = NULL; + Oid oid = InvalidOid; ConversionLocation errpos; ErrorContextCallback errcallback; MemoryContext oldcontext; @@ -4431,7 +4432,11 @@ make_tuple_from_result_row(PGresult *res, else valstr = PQgetvalue(res, row, j); - /* convert value to internal representation */ + /* + * convert value to internal representation + * + * Note: we ignore system columns other than ctid and oid in result + */ errpos.cur_attno = i; if (i > 0) { @@ -4446,7 +4451,7 @@ make_tuple_from_result_row(PGresult *res, } else if (i == SelfItemPointerAttributeNumber) { - /* ctid --- note we ignore any other system column in result */ + /* ctid */ if (valstr != NULL) { Datum datum; @@ -4455,6 +4460,17 @@ make_tuple_from_result_row(PGresult *res, ctid = (ItemPointer) DatumGetPointer(datum); } } + else if (i == ObjectIdAttributeNumber) + { + /* oid */ + if (valstr != NULL) + { + Datum datum; + + datum = DirectFunctionCall1(oidin, CStringGetDatum(valstr)); + oid = DatumGetObjectId(datum); + } + } errpos.cur_attno = 0; j++; @@ -4498,6 +4514,12 @@ make_tuple_from_result_row(PGresult *res, HeapTupleHeaderSetXmin(tuple->t_data, InvalidTransactionId); HeapTupleHeaderSetCmin(tuple->t_data, InvalidTransactionId); + /* + * If we have an OID to return, install it. + */ + if (OidIsValid(oid)) + HeapTupleSetOid(tuple, oid); + /* Clean up */ MemoryContextReset(temp_context); @@ -4525,6 +4547,8 @@ conversion_error_callback(void *arg) attname = NameStr(tupdesc->attrs[errpos->cur_attno - 1]->attname); else if (errpos->cur_attno == SelfItemPointerAttributeNumber) attname = "ctid"; + else if (errpos->cur_attno == ObjectIdAttributeNumber) + attname = "oid"; relname = RelationGetRelationName(errpos->rel); } diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 6f684a1b0c..4f68e8904e 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -136,6 +136,14 @@ CREATE FOREIGN TABLE ft6 ( c3 text ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4'); +-- A table with oids. CREATE FOREIGN TABLE doesn't support the +-- WITH OIDS option, but ALTER does. +CREATE FOREIGN TABLE ft_pg_type ( + typname name, + typlen smallint +) SERVER loopback OPTIONS (schema_name 'pg_catalog', table_name 'pg_type'); +ALTER TABLE ft_pg_type SET WITH OIDS; + -- =================================================================== -- tests for validator -- =================================================================== @@ -577,7 +585,7 @@ DEALLOCATE st3; DEALLOCATE st4; DEALLOCATE st5; --- System columns, except ctid, should not be sent to remote +-- System columns, except ctid and oid, should not be sent to remote EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE t1.tableoid = 'pg_class'::regclass LIMIT 1; SELECT * FROM ft1 t1 WHERE t1.tableoid = 'ft1'::regclass LIMIT 1; @@ -590,6 +598,9 @@ SELECT * FROM ft1 t1 WHERE t1.ctid = '(0,2)'; EXPLAIN (VERBOSE, COSTS OFF) SELECT ctid, * FROM ft1 t1 LIMIT 1; SELECT ctid, * FROM ft1 t1 LIMIT 1; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT oid, * FROM ft_pg_type WHERE typname = 'int4'; +SELECT oid, * FROM ft_pg_type WHERE typname = 'int4'; -- =================================================================== -- used in pl/pgsql function From fbf28b6b52c269188262a87247adb2c359acd6c5 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 26 Aug 2016 10:07:28 -0400 Subject: [PATCH 071/871] Fix logic for adding "parallel worker" context line to worker errors. The previous coding here was capable of adding a "parallel worker" context line to errors that were not, in fact, returned from a parallel worker. Instead of using an errcontext callback to add that annotation, just paste it onto the message by hand; this looks uglier but is more reliable. Discussion: <19757.1472151987@sss.pgh.pa.us> --- src/backend/access/transam/parallel.c | 54 +++++++++++++-------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c index a47eba647b..ec6e1c5e6d 100644 --- a/src/backend/access/transam/parallel.c +++ b/src/backend/access/transam/parallel.c @@ -108,7 +108,6 @@ static dlist_head pcxt_list = DLIST_STATIC_INIT(pcxt_list); /* Private functions. */ static void HandleParallelMessage(ParallelContext *pcxt, int i, StringInfo msg); -static void ParallelErrorContext(void *arg); static void ParallelExtensionTrampoline(dsm_segment *seg, shm_toc *toc); static void ParallelWorkerMain(Datum main_arg); static void WaitForParallelWorkersToExit(ParallelContext *pcxt); @@ -788,30 +787,43 @@ HandleParallelMessage(ParallelContext *pcxt, int i, StringInfo msg) case 'N': /* NoticeResponse */ { ErrorData edata; - ErrorContextCallback errctx; ErrorContextCallback *save_error_context_stack; - /* - * Rethrow the error using the error context callbacks that - * were in effect when the context was created, not the - * current ones. - */ - save_error_context_stack = error_context_stack; - errctx.callback = ParallelErrorContext; - errctx.arg = NULL; - errctx.previous = pcxt->error_context_stack; - error_context_stack = &errctx; - /* Parse ErrorResponse or NoticeResponse. */ pq_parse_errornotice(msg, &edata); /* Death of a worker isn't enough justification for suicide. */ edata.elevel = Min(edata.elevel, ERROR); - /* Rethrow error or notice. */ + /* + * If desired, add a context line to show that this is a + * message propagated from a parallel worker. Otherwise, it + * can sometimes be confusing to understand what actually + * happened. (We don't do this in FORCE_PARALLEL_REGRESS mode + * because it causes test-result instability depending on + * whether a parallel worker is actually used or not.) + */ + if (force_parallel_mode != FORCE_PARALLEL_REGRESS) + { + if (edata.context) + edata.context = psprintf("%s\n%s", edata.context, + _("parallel worker")); + else + edata.context = pstrdup(_("parallel worker")); + } + + /* + * Context beyond that should use the error context callbacks + * that were in effect when the ParallelContext was created, + * not the current ones. + */ + save_error_context_stack = error_context_stack; + error_context_stack = pcxt->error_context_stack; + + /* Rethrow error or print notice. */ ThrowErrorData(&edata); - /* Restore previous context. */ + /* Not an error, so restore previous context stack. */ error_context_stack = save_error_context_stack; break; @@ -1112,18 +1124,6 @@ ParallelExtensionTrampoline(dsm_segment *seg, shm_toc *toc) entrypt(seg, toc); } -/* - * Give the user a hint that this is a message propagated from a parallel - * worker. Otherwise, it can sometimes be confusing to understand what - * actually happened. - */ -static void -ParallelErrorContext(void *arg) -{ - if (force_parallel_mode != FORCE_PARALLEL_REGRESS) - errcontext("parallel worker"); -} - /* * Update shared memory with the ending location of the last WAL record we * wrote, if it's greater than the value already stored there. From 8529036b53298c0555670b4a81ed7349c44aeeb4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 26 Aug 2016 14:15:47 -0400 Subject: [PATCH 072/871] Fix assorted small bugs in ThrowErrorData(). Copy the palloc'd strings into the correct context, ie ErrorContext not wherever the source ErrorData is. This would be a large bug, except that it appears that all catchers of thrown errors do either EmitErrorReport or CopyErrorData before doing anything that would cause transient memory contexts to be cleaned up. Still, it's wrong and it will bite somebody someday. Fix failure to copy cursorpos and internalpos. Utter the appropriate incantations involving recursion_depth, so that we'll behave sanely if we get an error inside pstrdup. (In general, the body of this function ought to act like, eg, errdetail().) Per code reading induced by Jakob Egger's report. --- src/backend/utils/error/elog.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 78d441d198..86e0cd9315 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -1601,7 +1601,10 @@ FlushErrorState(void) /* * ThrowErrorData --- report an error described by an ErrorData structure * - * This is intended to be used to re-report errors originally thrown by + * This is somewhat like ReThrowError, but it allows elevels besides ERROR, + * and the boolean flags such as output_to_server are computed via the + * default rules rather than being copied from the given ErrorData. + * This is primarily used to re-report errors originally reported by * background worker processes and then propagated (with or without * modification) to the backend responsible for them. */ @@ -1613,13 +1616,14 @@ ThrowErrorData(ErrorData *edata) if (!errstart(edata->elevel, edata->filename, edata->lineno, edata->funcname, NULL)) - return; + return; /* error is not to be reported at all */ newedata = &errordata[errordata_stack_depth]; - oldcontext = MemoryContextSwitchTo(edata->assoc_context); + recursion_depth++; + oldcontext = MemoryContextSwitchTo(newedata->assoc_context); - /* Copy the supplied fields to the error stack. */ - if (edata->sqlerrcode > 0) + /* Copy the supplied fields to the error stack entry. */ + if (edata->sqlerrcode != 0) newedata->sqlerrcode = edata->sqlerrcode; if (edata->message) newedata->message = pstrdup(edata->message); @@ -1631,6 +1635,7 @@ ThrowErrorData(ErrorData *edata) newedata->hint = pstrdup(edata->hint); if (edata->context) newedata->context = pstrdup(edata->context); + /* assume message_id is not available */ if (edata->schema_name) newedata->schema_name = pstrdup(edata->schema_name); if (edata->table_name) @@ -1641,11 +1646,15 @@ ThrowErrorData(ErrorData *edata) newedata->datatype_name = pstrdup(edata->datatype_name); if (edata->constraint_name) newedata->constraint_name = pstrdup(edata->constraint_name); + newedata->cursorpos = edata->cursorpos; + newedata->internalpos = edata->internalpos; if (edata->internalquery) newedata->internalquery = pstrdup(edata->internalquery); MemoryContextSwitchTo(oldcontext); + recursion_depth--; + /* Process the error. */ errfinish(0); } From 45a36e68539dcd7095a257b49f6f38ae77dec30d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 26 Aug 2016 14:19:03 -0400 Subject: [PATCH 073/871] Put static forward declarations in elog.c back into same order as code. The guiding principle for the last few patches in this area apparently involved throwing darts. Cosmetic only, but back-patch to 9.6 because there is no reason for 9.6 and HEAD to diverge yet in this file. --- src/backend/utils/error/elog.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 86e0cd9315..03c4a39761 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -79,11 +79,10 @@ #include "utils/ps_status.h" +/* In this module, access gettext() via err_gettext() */ #undef _ #define _(x) err_gettext(x) -static const char *err_gettext(const char *str) pg_attribute_format_arg(1); -static void set_errdata_field(MemoryContextData *cxt, char **ptr, const char *str); /* Global variables */ ErrorContextCallback *error_context_stack = NULL; @@ -129,10 +128,9 @@ static int syslog_facility = LOG_LOCAL0; static void write_syslog(int level, const char *line); #endif -static void write_console(const char *line, int len); - #ifdef WIN32 extern char *event_source; + static void write_eventlog(int level, const char *line, int len); #endif @@ -149,7 +147,6 @@ static int recursion_depth = 0; /* to detect actual recursion */ * Saved timeval and buffers for formatted timestamps that might be used by * both log_line_prefix and csv logs. */ - static struct timeval saved_timeval; static bool saved_timeval_set = false; @@ -169,9 +166,16 @@ static char formatted_log_time[FORMATTED_TS_LEN]; } while (0) +static const char *err_gettext(const char *str) pg_attribute_format_arg(1); +static void set_errdata_field(MemoryContextData *cxt, char **ptr, const char *str); +static void write_console(const char *line, int len); +static void setup_formatted_log_time(void); +static void setup_formatted_start_time(void); static const char *process_log_prefix_padding(const char *p, int *padding); static void log_line_prefix(StringInfo buf, ErrorData *edata); +static void write_csvlog(ErrorData *edata); static void send_message_to_server_log(ErrorData *edata); +static void write_pipe_chunks(char *data, int len, int dest); static void send_message_to_frontend(ErrorData *edata); static char *expand_fmt_string(const char *fmt, ErrorData *edata); static const char *useful_strerror(int errnum); @@ -179,10 +183,6 @@ static const char *get_errno_symbol(int errnum); static const char *error_severity(int elevel); static void append_with_tabs(StringInfo buf, const char *str); static bool is_log_level_output(int elevel, int log_min_level); -static void write_pipe_chunks(char *data, int len, int dest); -static void write_csvlog(ErrorData *edata); -static void setup_formatted_log_time(void); -static void setup_formatted_start_time(void); /* From 78dcd027e8f7ed213f69da932853dc4b7cb9cb44 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 26 Aug 2016 15:04:05 -0400 Subject: [PATCH 074/871] Fix potential memory leakage from HandleParallelMessages(). HandleParallelMessages leaked memory into the caller's context. Since it's called from ProcessInterrupts, there is basically zero certainty as to what CurrentMemoryContext is, which means we could be leaking into long-lived contexts. Over the processing of many worker messages that would grow to be a problem. Things could be even worse than just a leak, if we happened to service the interrupt while ErrorContext is current: elog.c thinks it can reset that on its own whim, possibly yanking storage out from under HandleParallelMessages. Give HandleParallelMessages its own dedicated context instead, which we can reset during each call to ensure there's no accumulation of wasted memory. Discussion: <16610.1472222135@sss.pgh.pa.us> --- src/backend/access/transam/parallel.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c index ec6e1c5e6d..949bfb8b3e 100644 --- a/src/backend/access/transam/parallel.c +++ b/src/backend/access/transam/parallel.c @@ -702,6 +702,9 @@ void HandleParallelMessages(void) { dlist_iter iter; + MemoryContext oldcontext; + + static MemoryContext hpm_context = NULL; /* * This is invoked from ProcessInterrupts(), and since some of the @@ -712,6 +715,23 @@ HandleParallelMessages(void) */ HOLD_INTERRUPTS(); + /* + * Moreover, CurrentMemoryContext might be pointing almost anywhere. We + * don't want to risk leaking data into long-lived contexts, so let's do + * our work here in a private context that we can reset on each use. + */ + if (hpm_context == NULL) /* first time through? */ + hpm_context = AllocSetContextCreate(TopMemoryContext, + "HandleParallelMessages context", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + else + MemoryContextReset(hpm_context); + + oldcontext = MemoryContextSwitchTo(hpm_context); + + /* OK to process messages. Reset the flag saying there are more to do. */ ParallelMessagePending = false; dlist_foreach(iter, &pcxt_list) @@ -758,6 +778,11 @@ HandleParallelMessages(void) } } + MemoryContextSwitchTo(oldcontext); + + /* Might as well clear the context on our way out */ + MemoryContextReset(hpm_context); + RESUME_INTERRUPTS(); } From 26fa446da64716f12ab3a623434c644fcb344b2e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 26 Aug 2016 16:20:17 -0400 Subject: [PATCH 075/871] Add a nonlocalized version of the severity field to client error messages. This has been requested a few times, but the use-case for it was never entirely clear. The reason for adding it now is that transmission of error reports from parallel workers fails when NLS is active, because pq_parse_errornotice() wrongly assumes that the existing severity field is nonlocalized. There are other ways we could have fixed that, but the other options were basically kluges, whereas this way provides something that's at least arguably a useful feature along with the bug fix. Per report from Jakob Egger. Back-patch into 9.6, because otherwise parallel query is essentially unusable in non-English locales. The problem exists in 9.5 as well, but we don't want to risk changing on-the-wire behavior in 9.5 (even though the possibility of new error fields is specifically called out in the protocol document). It may be sufficient to leave the issue unfixed in 9.5, given the very limited usefulness of pq_parse_errornotice in that version. Discussion: --- doc/src/sgml/libpq.sgml | 16 ++++++++++++++++ doc/src/sgml/protocol.sgml | 19 +++++++++++++++++++ src/backend/libpq/pqmq.c | 26 +++++++++++++++++++++----- src/backend/utils/error/elog.c | 33 ++++++++++++++++++++------------- src/include/postgres_ext.h | 1 + src/interfaces/libpq/fe-exec.c | 1 + 6 files changed, 78 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index f22e3da047..2f9350b10e 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -2767,6 +2767,22 @@ char *PQresultErrorField(const PGresult *res, int fieldcode); + + PG_DIAG_SEVERITY_NONLOCALIZED + + + The severity; the field contents are ERROR, + FATAL, or PANIC (in an error message), + or WARNING, NOTICE, DEBUG, + INFO, or LOG (in a notice message). + This is identical to the PG_DIAG_SEVERITY field except + that the contents are never localized. This is present only in + reports generated by PostgreSQL versions 9.6 + and later. + + + + PG_DIAG_SQLSTATE diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 9c96d8fc44..68b0941029 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -4882,6 +4882,25 @@ message. + + +V + + + + Severity: the field contents are + ERROR, FATAL, or + PANIC (in an error message), or + WARNING, NOTICE, DEBUG, + INFO, or LOG (in a notice message). + This is identical to the S field except + that the contents are never localized. This is present only in + messages generated by PostgreSQL versions 9.6 + and later. + + + + C diff --git a/src/backend/libpq/pqmq.c b/src/backend/libpq/pqmq.c index 921242fbc4..bfe66c6c44 100644 --- a/src/backend/libpq/pqmq.c +++ b/src/backend/libpq/pqmq.c @@ -237,10 +237,26 @@ pq_parse_errornotice(StringInfo msg, ErrorData *edata) switch (code) { case PG_DIAG_SEVERITY: + /* ignore, trusting we'll get a nonlocalized version */ + break; + case PG_DIAG_SEVERITY_NONLOCALIZED: if (strcmp(value, "DEBUG") == 0) - edata->elevel = DEBUG1; /* or some other DEBUG level */ + { + /* + * We can't reconstruct the exact DEBUG level, but + * presumably it was >= client_min_messages, so select + * DEBUG1 to ensure we'll pass it on to the client. + */ + edata->elevel = DEBUG1; + } else if (strcmp(value, "LOG") == 0) - edata->elevel = LOG; /* can't be COMMERROR */ + { + /* + * It can't be LOG_SERVER_ONLY, or the worker wouldn't + * have sent it to us; so LOG is the correct value. + */ + edata->elevel = LOG; + } else if (strcmp(value, "INFO") == 0) edata->elevel = INFO; else if (strcmp(value, "NOTICE") == 0) @@ -254,11 +270,11 @@ pq_parse_errornotice(StringInfo msg, ErrorData *edata) else if (strcmp(value, "PANIC") == 0) edata->elevel = PANIC; else - elog(ERROR, "unknown error severity"); + elog(ERROR, "unrecognized error severity: \"%s\"", value); break; case PG_DIAG_SQLSTATE: if (strlen(value) != 5) - elog(ERROR, "malformed sql state"); + elog(ERROR, "invalid SQLSTATE: \"%s\"", value); edata->sqlerrcode = MAKE_SQLSTATE(value[0], value[1], value[2], value[3], value[4]); break; @@ -308,7 +324,7 @@ pq_parse_errornotice(StringInfo msg, ErrorData *edata) edata->funcname = pstrdup(value); break; default: - elog(ERROR, "unknown error field: %d", (int) code); + elog(ERROR, "unrecognized error field code: %d", (int) code); break; } } diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 03c4a39761..224ee7801c 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -2753,7 +2753,7 @@ write_csvlog(ErrorData *edata) appendStringInfoChar(&buf, ','); /* Error severity */ - appendStringInfoString(&buf, error_severity(edata->elevel)); + appendStringInfoString(&buf, _(error_severity(edata->elevel))); appendStringInfoChar(&buf, ','); /* SQL state code */ @@ -2870,7 +2870,7 @@ send_message_to_server_log(ErrorData *edata) formatted_log_time[0] = '\0'; log_line_prefix(&buf, edata); - appendStringInfo(&buf, "%s: ", error_severity(edata->elevel)); + appendStringInfo(&buf, "%s: ", _(error_severity(edata->elevel))); if (Log_error_verbosity >= PGERROR_VERBOSE) appendStringInfo(&buf, "%s: ", unpack_sql_state(edata->sqlerrcode)); @@ -3153,12 +3153,16 @@ send_message_to_frontend(ErrorData *edata) if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3) { /* New style with separate fields */ + const char *sev; char tbuf[12]; int ssval; int i; + sev = error_severity(edata->elevel); pq_sendbyte(&msgbuf, PG_DIAG_SEVERITY); - err_sendstring(&msgbuf, error_severity(edata->elevel)); + err_sendstring(&msgbuf, _(sev)); + pq_sendbyte(&msgbuf, PG_DIAG_SEVERITY_NONLOCALIZED); + err_sendstring(&msgbuf, sev); /* unpack MAKE_SQLSTATE code */ ssval = edata->sqlerrcode; @@ -3277,7 +3281,7 @@ send_message_to_frontend(ErrorData *edata) initStringInfo(&buf); - appendStringInfo(&buf, "%s: ", error_severity(edata->elevel)); + appendStringInfo(&buf, "%s: ", _(error_severity(edata->elevel))); if (edata->show_funcname && edata->funcname) appendStringInfo(&buf, "%s: ", edata->funcname); @@ -3587,7 +3591,10 @@ get_errno_symbol(int errnum) /* - * error_severity --- get localized string representing elevel + * error_severity --- get string representing elevel + * + * The string is not localized here, but we mark the strings for translation + * so that callers can invoke _() on the result. */ static const char * error_severity(int elevel) @@ -3601,29 +3608,29 @@ error_severity(int elevel) case DEBUG3: case DEBUG4: case DEBUG5: - prefix = _("DEBUG"); + prefix = gettext_noop("DEBUG"); break; case LOG: case LOG_SERVER_ONLY: - prefix = _("LOG"); + prefix = gettext_noop("LOG"); break; case INFO: - prefix = _("INFO"); + prefix = gettext_noop("INFO"); break; case NOTICE: - prefix = _("NOTICE"); + prefix = gettext_noop("NOTICE"); break; case WARNING: - prefix = _("WARNING"); + prefix = gettext_noop("WARNING"); break; case ERROR: - prefix = _("ERROR"); + prefix = gettext_noop("ERROR"); break; case FATAL: - prefix = _("FATAL"); + prefix = gettext_noop("FATAL"); break; case PANIC: - prefix = _("PANIC"); + prefix = gettext_noop("PANIC"); break; default: prefix = "???"; diff --git a/src/include/postgres_ext.h b/src/include/postgres_ext.h index 74c344c704..ae2f087798 100644 --- a/src/include/postgres_ext.h +++ b/src/include/postgres_ext.h @@ -49,6 +49,7 @@ typedef PG_INT64_TYPE pg_int64; * applications. */ #define PG_DIAG_SEVERITY 'S' +#define PG_DIAG_SEVERITY_NONLOCALIZED 'V' #define PG_DIAG_SQLSTATE 'C' #define PG_DIAG_MESSAGE_PRIMARY 'M' #define PG_DIAG_MESSAGE_DETAIL 'D' diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c index d1b91c841c..a9ba54628f 100644 --- a/src/interfaces/libpq/fe-exec.c +++ b/src/interfaces/libpq/fe-exec.c @@ -824,6 +824,7 @@ pqInternalNotice(const PGNoticeHooks *hooks, const char *fmt,...) */ pqSaveMessageField(res, PG_DIAG_MESSAGE_PRIMARY, msgBuf); pqSaveMessageField(res, PG_DIAG_SEVERITY, libpq_gettext("NOTICE")); + pqSaveMessageField(res, PG_DIAG_SEVERITY_NONLOCALIZED, "NOTICE"); /* XXX should provide a SQLSTATE too? */ /* From ea268cdc9a2631da4a5748b00059a9fd43470d0e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 27 Aug 2016 17:50:38 -0400 Subject: [PATCH 076/871] Add macros to make AllocSetContextCreate() calls simpler and safer. I found that half a dozen (nearly 5%) of our AllocSetContextCreate calls had typos in the context-sizing parameters. While none of these led to especially significant problems, they did create minor inefficiencies, and it's now clear that expecting people to copy-and-paste those calls accurately is not a great idea. Let's reduce the risk of future errors by introducing single macros that encapsulate the common use-cases. Three such macros are enough to cover all but two special-purpose contexts; those two calls can be left as-is, I think. While this patch doesn't in itself improve matters for third-party extensions, it doesn't break anything for them either, and they can gradually adopt the simplified notation over time. In passing, change TopMemoryContext to use the default allocation parameters. Formerly it could only be extended 8K at a time. That was probably reasonable when this code was written; but nowadays we create many more contexts than we did then, so that it's not unusual to have a couple hundred K in TopMemoryContext, even without considering various dubious code that sticks other things there. There seems no good reason not to let it use growing blocks like most other contexts. Back-patch to 9.6, mostly because that's still close enough to HEAD that it's easy to do so, and keeping the branches in sync can be expected to avoid some future back-patching pain. The bugs fixed by these changes don't seem to be significant enough to justify fixing them further back. Discussion: <21072.1472321324@sss.pgh.pa.us> --- contrib/bloom/blinsert.c | 8 ++--- contrib/dblink/dblink.c | 4 +-- contrib/file_fdw/file_fdw.c | 4 +-- contrib/pg_trgm/trgm_regexp.c | 4 +-- contrib/postgres_fdw/postgres_fdw.c | 20 +++-------- contrib/sepgsql/uavc.c | 6 ++-- contrib/test_decoding/test_decoding.c | 4 +-- src/backend/access/brin/brin.c | 16 +++------ src/backend/access/brin/brin_tuple.c | 4 +-- src/backend/access/common/printtup.c | 4 +-- src/backend/access/gin/ginbtree.c | 4 +-- src/backend/access/gin/ginfast.c | 4 +-- src/backend/access/gin/gininsert.c | 12 ++----- src/backend/access/gin/ginscan.c | 8 ++--- src/backend/access/gin/ginvacuum.c | 4 +-- src/backend/access/gin/ginxlog.c | 5 ++- src/backend/access/gist/gist.c | 8 ++--- src/backend/access/gist/gistscan.c | 8 ++--- src/backend/access/heap/rewriteheap.c | 4 +-- src/backend/access/nbtree/nbtree.c | 4 +-- src/backend/access/nbtree/nbtutils.c | 6 ++-- src/backend/access/spgist/spginsert.c | 8 ++--- src/backend/access/spgist/spgscan.c | 4 +-- src/backend/access/spgist/spgxlog.c | 4 +-- src/backend/access/transam/multixact.c | 6 ++-- src/backend/access/transam/parallel.c | 12 +++---- src/backend/access/transam/xact.c | 8 ++--- src/backend/access/transam/xlog.c | 4 +-- src/backend/access/transam/xloginsert.c | 4 +-- src/backend/bootstrap/bootstrap.c | 4 +-- src/backend/catalog/objectaddress.c | 4 +-- src/backend/commands/analyze.c | 12 ++----- src/backend/commands/cluster.c | 4 +-- src/backend/commands/copy.c | 8 ++--- src/backend/commands/event_trigger.c | 8 ++--- src/backend/commands/indexcmds.c | 4 +-- src/backend/commands/policy.c | 4 +-- src/backend/commands/trigger.c | 8 ++--- src/backend/commands/vacuum.c | 4 +-- src/backend/executor/execUtils.c | 12 ++----- src/backend/executor/functions.c | 4 +-- src/backend/executor/nodeFunctionscan.c | 4 +-- src/backend/executor/nodeHash.c | 8 ++--- src/backend/executor/nodeRecursiveunion.c | 8 ++--- src/backend/executor/nodeSetOp.c | 8 ++--- src/backend/executor/nodeSubplan.c | 8 ++--- src/backend/executor/nodeUnique.c | 4 +-- src/backend/executor/nodeWindowAgg.c | 18 ++++------ src/backend/executor/spi.c | 23 ++++-------- src/backend/executor/tqueue.c | 4 +-- src/backend/libpq/be-fsstubs.c | 4 +-- src/backend/libpq/hba.c | 14 +++----- src/backend/optimizer/geqo/geqo_eval.c | 4 +-- src/backend/optimizer/util/clauses.c | 8 ++--- src/backend/postmaster/autovacuum.c | 24 ++++--------- src/backend/postmaster/bgwriter.c | 4 +-- src/backend/postmaster/checkpointer.c | 4 +-- src/backend/postmaster/pgstat.c | 4 +-- src/backend/postmaster/postmaster.c | 4 +-- src/backend/postmaster/walwriter.c | 4 +-- src/backend/replication/logical/logical.c | 6 ++-- .../replication/logical/reorderbuffer.c | 6 ++-- src/backend/replication/logical/snapbuild.c | 4 +-- src/backend/replication/walsender.c | 4 +-- src/backend/storage/buffer/localbuf.c | 4 +-- src/backend/storage/file/reinit.c | 4 +-- src/backend/storage/lmgr/lwlock.c | 4 +-- src/backend/storage/smgr/md.c | 10 ++---- src/backend/tcop/postgres.c | 8 ++--- src/backend/tsearch/spell.c | 4 +-- src/backend/utils/adt/array_expanded.c | 4 +-- src/backend/utils/adt/arrayfuncs.c | 8 ++--- src/backend/utils/adt/jsonfuncs.c | 16 +++------ src/backend/utils/adt/xml.c | 6 ++-- src/backend/utils/cache/catcache.c | 4 +-- src/backend/utils/cache/evtcache.c | 4 +-- src/backend/utils/cache/plancache.c | 35 ++++++------------- src/backend/utils/cache/relcache.c | 18 +++------- src/backend/utils/cache/ts_cache.c | 4 +-- src/backend/utils/cache/typcache.c | 8 ++--- src/backend/utils/fmgr/funcapi.c | 4 +-- src/backend/utils/hash/dynahash.c | 4 +-- src/backend/utils/init/postinit.c | 4 +-- src/backend/utils/misc/guc-file.l | 4 +-- src/backend/utils/misc/tzparser.c | 4 +-- src/backend/utils/mmgr/aset.c | 6 +++- src/backend/utils/mmgr/mcxt.c | 11 +++--- src/backend/utils/mmgr/portalmem.c | 12 ++----- src/backend/utils/sort/tuplesort.c | 8 ++--- src/include/utils/memutils.h | 14 +++++++- src/pl/plperl/plperl.c | 12 ++----- src/pl/plpgsql/src/pl_comp.c | 10 ++---- src/pl/plpgsql/src/pl_exec.c | 8 ++--- src/pl/plpython/plpy_cursorobject.c | 8 ++--- src/pl/plpython/plpy_main.c | 8 ++--- src/pl/plpython/plpy_procedure.c | 4 +-- src/pl/plpython/plpy_spi.c | 8 ++--- src/pl/plpython/plpy_typeio.c | 8 ++--- src/pl/tcl/pltcl.c | 4 +-- 99 files changed, 206 insertions(+), 522 deletions(-) diff --git a/contrib/bloom/blinsert.c b/contrib/bloom/blinsert.c index 78eec5c67e..0946aa29ec 100644 --- a/contrib/bloom/blinsert.c +++ b/contrib/bloom/blinsert.c @@ -130,9 +130,7 @@ blbuild(Relation heap, Relation index, IndexInfo *indexInfo) initBloomState(&buildstate.blstate, index); buildstate.tmpCtx = AllocSetContextCreate(CurrentMemoryContext, "Bloom build temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); initCachedPage(&buildstate); /* Do the heap scan */ @@ -204,9 +202,7 @@ blinsert(Relation index, Datum *values, bool *isnull, insertCtx = AllocSetContextCreate(CurrentMemoryContext, "Bloom insert temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldCtx = MemoryContextSwitchTo(insertCtx); diff --git a/contrib/dblink/dblink.c b/contrib/dblink/dblink.c index 9c8e308358..d4f9090f06 100644 --- a/contrib/dblink/dblink.c +++ b/contrib/dblink/dblink.c @@ -980,9 +980,7 @@ materializeQueryResult(FunctionCallInfo fcinfo, /* Create short-lived memory context for data conversions */ sinfo.tmpcontext = AllocSetContextCreate(CurrentMemoryContext, "dblink temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* execute query, collecting any tuples into the tuplestore */ res = storeQueryResult(&sinfo, conn, sql); diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index c0491318c0..b42de873e0 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -1061,9 +1061,7 @@ file_acquire_sample_rows(Relation onerel, int elevel, */ tupcontext = AllocSetContextCreate(CurrentMemoryContext, "file_fdw temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* Prepare for sampling rows */ reservoir_init_selection_state(&rstate, targrows); diff --git a/contrib/pg_trgm/trgm_regexp.c b/contrib/pg_trgm/trgm_regexp.c index 3f09a9c718..005701fcd9 100644 --- a/contrib/pg_trgm/trgm_regexp.c +++ b/contrib/pg_trgm/trgm_regexp.c @@ -529,9 +529,7 @@ createTrgmNFA(text *text_re, Oid collation, */ tmpcontext = AllocSetContextCreate(CurrentMemoryContext, "createTrgmNFA temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcontext = MemoryContextSwitchTo(tmpcontext); /* diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index b92f29958f..daf0438532 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -1315,14 +1315,10 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags) /* Create contexts for batches of tuples and per-tuple temp workspace. */ fsstate->batch_cxt = AllocSetContextCreate(estate->es_query_cxt, "postgres_fdw tuple data", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); fsstate->temp_cxt = AllocSetContextCreate(estate->es_query_cxt, "postgres_fdw temporary data", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); /* * Get info we'll need for converting data fetched from the foreign server @@ -1695,9 +1691,7 @@ postgresBeginForeignModify(ModifyTableState *mtstate, /* Create context for per-tuple temp workspace. */ fmstate->temp_cxt = AllocSetContextCreate(estate->es_query_cxt, "postgres_fdw temporary data", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); /* Prepare for input conversion of RETURNING results. */ if (fmstate->has_returning) @@ -2294,9 +2288,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags) /* Create context for per-tuple temp workspace. */ dmstate->temp_cxt = AllocSetContextCreate(estate->es_query_cxt, "postgres_fdw temporary data", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); /* Prepare for input conversion of RETURNING results. */ if (dmstate->has_returning) @@ -3481,9 +3473,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel, astate.anl_cxt = CurrentMemoryContext; astate.temp_cxt = AllocSetContextCreate(CurrentMemoryContext, "postgres_fdw temporary data", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); /* * Get the connection to use. We do the remote access as the table's diff --git a/contrib/sepgsql/uavc.c b/contrib/sepgsql/uavc.c index 10fa9a0b0b..6e358dbef7 100644 --- a/contrib/sepgsql/uavc.c +++ b/contrib/sepgsql/uavc.c @@ -498,13 +498,11 @@ sepgsql_avc_init(void) int rc; /* - * All the avc stuff shall be allocated on avc_mem_cxt + * All the avc stuff shall be allocated in avc_mem_cxt */ avc_mem_cxt = AllocSetContextCreate(TopMemoryContext, "userspace access vector cache", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); memset(avc_slots, 0, sizeof(avc_slots)); avc_num_caches = 0; avc_lru_hint = 0; diff --git a/contrib/test_decoding/test_decoding.c b/contrib/test_decoding/test_decoding.c index c3508f0e13..949e9a78d9 100644 --- a/contrib/test_decoding/test_decoding.c +++ b/contrib/test_decoding/test_decoding.c @@ -102,9 +102,7 @@ pg_decode_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, data = palloc0(sizeof(TestDecodingData)); data->context = AllocSetContextCreate(ctx->context, "text conversion context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); data->include_xids = true; data->include_timestamp = false; data->skip_empty_xacts = false; diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index b194d33cc5..1b45a4c901 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -165,9 +165,7 @@ brininsert(Relation idxRel, Datum *values, bool *nulls, bdesc = brin_build_desc(idxRel); tupcxt = AllocSetContextCreate(CurrentMemoryContext, "brininsert cxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(tupcxt); } @@ -347,9 +345,7 @@ bringetbitmap(IndexScanDesc scan, TIDBitmap *tbm) */ perRangeCxt = AllocSetContextCreate(CurrentMemoryContext, "bringetbitmap cxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(perRangeCxt); /* @@ -856,9 +852,7 @@ brin_build_desc(Relation rel) cxt = AllocSetContextCreate(CurrentMemoryContext, "brin desc cxt", - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); oldcxt = MemoryContextSwitchTo(cxt); tupdesc = RelationGetDescr(rel); @@ -1169,9 +1163,7 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b) /* Use our own memory context to avoid retail pfree */ cxt = AllocSetContextCreate(CurrentMemoryContext, "brin union", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(cxt); db = brin_deform_tuple(bdesc, b); MemoryContextSwitchTo(oldcxt); diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c index 64b8264959..3caec14d17 100644 --- a/src/backend/access/brin/brin_tuple.c +++ b/src/backend/access/brin/brin_tuple.c @@ -367,9 +367,7 @@ brin_new_memtuple(BrinDesc *brdesc) dtup->bt_context = AllocSetContextCreate(CurrentMemoryContext, "brin dtuple", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); return dtup; } diff --git a/src/backend/access/common/printtup.c b/src/backend/access/common/printtup.c index d9664aa6c6..d213af9074 100644 --- a/src/backend/access/common/printtup.c +++ b/src/backend/access/common/printtup.c @@ -135,9 +135,7 @@ printtup_startup(DestReceiver *self, int operation, TupleDesc typeinfo) */ myState->tmpcontext = AllocSetContextCreate(CurrentMemoryContext, "printtup", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); if (PG_PROTOCOL_MAJOR(FrontendProtocol) < 3) { diff --git a/src/backend/access/gin/ginbtree.c b/src/backend/access/gin/ginbtree.c index fa383719e6..a0afec4f3c 100644 --- a/src/backend/access/gin/ginbtree.c +++ b/src/backend/access/gin/ginbtree.c @@ -348,9 +348,7 @@ ginPlaceToPage(GinBtree btree, GinBtreeStack *stack, */ tmpCxt = AllocSetContextCreate(CurrentMemoryContext, "ginPlaceToPage temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldCxt = MemoryContextSwitchTo(tmpCxt); if (GinPageIsData(page)) diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c index 59a63f28d0..6b709dbdb3 100644 --- a/src/backend/access/gin/ginfast.c +++ b/src/backend/access/gin/ginfast.c @@ -808,9 +808,7 @@ ginInsertCleanup(GinState *ginstate, bool full_clean, */ opCtx = AllocSetContextCreate(CurrentMemoryContext, "GIN insert cleanup temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldCtx = MemoryContextSwitchTo(opCtx); diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c index 9f784bf48d..4e09f76eb2 100644 --- a/src/backend/access/gin/gininsert.c +++ b/src/backend/access/gin/gininsert.c @@ -372,9 +372,7 @@ ginbuild(Relation heap, Relation index, IndexInfo *indexInfo) */ buildstate.tmpCtx = AllocSetContextCreate(CurrentMemoryContext, "Gin build temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * create a temporary memory context that is used for calling @@ -382,9 +380,7 @@ ginbuild(Relation heap, Relation index, IndexInfo *indexInfo) */ buildstate.funcCtx = AllocSetContextCreate(CurrentMemoryContext, "Gin build temporary context for user-defined function", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); buildstate.accum.ginstate = &buildstate.ginstate; ginInitBA(&buildstate.accum); @@ -495,9 +491,7 @@ gininsert(Relation index, Datum *values, bool *isnull, insertCtx = AllocSetContextCreate(CurrentMemoryContext, "Gin insert temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldCtx = MemoryContextSwitchTo(insertCtx); diff --git a/src/backend/access/gin/ginscan.c b/src/backend/access/gin/ginscan.c index c449c1cbc0..bfa86b521d 100644 --- a/src/backend/access/gin/ginscan.c +++ b/src/backend/access/gin/ginscan.c @@ -38,14 +38,10 @@ ginbeginscan(Relation rel, int nkeys, int norderbys) so->nkeys = 0; so->tempCtx = AllocSetContextCreate(CurrentMemoryContext, "Gin scan temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); so->keyCtx = AllocSetContextCreate(CurrentMemoryContext, "Gin scan key context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); initGinState(&so->ginstate, scan->indexRelation); scan->opaque = so; diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c index c258478f23..2685a1c373 100644 --- a/src/backend/access/gin/ginvacuum.c +++ b/src/backend/access/gin/ginvacuum.c @@ -526,9 +526,7 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext, "Gin vacuum temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); gvs.index = index; gvs.callback = callback; gvs.callback_state = callback_state; diff --git a/src/backend/access/gin/ginxlog.c b/src/backend/access/gin/ginxlog.c index b4d310f337..a40f1683dd 100644 --- a/src/backend/access/gin/ginxlog.c +++ b/src/backend/access/gin/ginxlog.c @@ -749,13 +749,12 @@ gin_xlog_startup(void) { opCtx = AllocSetContextCreate(CurrentMemoryContext, "GIN recovery temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } void gin_xlog_cleanup(void) { MemoryContextDelete(opCtx); + opCtx = NULL; } diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index 9a417ca2f4..f7f44b49aa 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -105,9 +105,7 @@ createTempGistContext(void) { return AllocSetContextCreate(CurrentMemoryContext, "GiST temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } /* @@ -1411,9 +1409,7 @@ initGISTstate(Relation index) /* Create the memory context that will hold the GISTSTATE */ scanCxt = AllocSetContextCreate(CurrentMemoryContext, "GiST scan context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldCxt = MemoryContextSwitchTo(scanCxt); /* Create and fill in the GISTSTATE */ diff --git a/src/backend/access/gist/gistscan.c b/src/backend/access/gist/gistscan.c index 6f07cd8d46..ba611ee490 100644 --- a/src/backend/access/gist/gistscan.c +++ b/src/backend/access/gist/gistscan.c @@ -140,9 +140,7 @@ gistrescan(IndexScanDesc scan, ScanKey key, int nkeys, /* second time through */ so->queueCxt = AllocSetContextCreate(so->giststate->scanCxt, "GiST queue context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); first_time = false; } else @@ -180,9 +178,7 @@ gistrescan(IndexScanDesc scan, ScanKey key, int nkeys, so->pageDataCxt = AllocSetContextCreate(so->giststate->scanCxt, "GiST page data context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } /* create new, empty RBTree for search queue */ diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c index f9ce9861e2..17584ba3ed 100644 --- a/src/backend/access/heap/rewriteheap.c +++ b/src/backend/access/heap/rewriteheap.c @@ -258,9 +258,7 @@ begin_heap_rewrite(Relation old_heap, Relation new_heap, TransactionId oldest_xm */ rw_cxt = AllocSetContextCreate(CurrentMemoryContext, "Table rewrite", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); old_cxt = MemoryContextSwitchTo(rw_cxt); /* Create and fill in the state struct */ diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index 4668c5ee59..128744c5b7 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -763,9 +763,7 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, /* Create a temporary memory context to run _bt_pagedel in */ vstate.pagedelcontext = AllocSetContextCreate(CurrentMemoryContext, "_bt_pagedel", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * The outer loop iterates over all index pages except the metapage, in diff --git a/src/backend/access/nbtree/nbtutils.c b/src/backend/access/nbtree/nbtutils.c index 5d335c7f97..063c988dc1 100644 --- a/src/backend/access/nbtree/nbtutils.c +++ b/src/backend/access/nbtree/nbtutils.c @@ -232,10 +232,8 @@ _bt_preprocess_array_keys(IndexScanDesc scan) */ if (so->arrayContext == NULL) so->arrayContext = AllocSetContextCreate(CurrentMemoryContext, - "BTree Array Context", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + "BTree array context", + ALLOCSET_SMALL_SIZES); else MemoryContextReset(so->arrayContext); diff --git a/src/backend/access/spgist/spginsert.c b/src/backend/access/spgist/spginsert.c index 44fd644e42..01c8d213f5 100644 --- a/src/backend/access/spgist/spginsert.c +++ b/src/backend/access/spgist/spginsert.c @@ -134,9 +134,7 @@ spgbuild(Relation heap, Relation index, IndexInfo *indexInfo) buildstate.tmpCtx = AllocSetContextCreate(CurrentMemoryContext, "SP-GiST build temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); reltuples = IndexBuildHeapScan(heap, index, indexInfo, true, spgistBuildCallback, (void *) &buildstate); @@ -213,9 +211,7 @@ spginsert(Relation index, Datum *values, bool *isnull, insertCtx = AllocSetContextCreate(CurrentMemoryContext, "SP-GiST insert temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldCtx = MemoryContextSwitchTo(insertCtx); initSpGistState(&spgstate, index); diff --git a/src/backend/access/spgist/spgscan.c b/src/backend/access/spgist/spgscan.c index 6f9e223f43..307c6a4ab5 100644 --- a/src/backend/access/spgist/spgscan.c +++ b/src/backend/access/spgist/spgscan.c @@ -193,9 +193,7 @@ spgbeginscan(Relation rel, int keysz, int orderbysz) initSpGistState(&so->state, scan->indexRelation); so->tempCxt = AllocSetContextCreate(CurrentMemoryContext, "SP-GiST search temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* Set up indexTupDesc and xs_itupdesc in case it's an index-only scan */ so->indexTupDesc = scan->xs_itupdesc = RelationGetDescr(rel); diff --git a/src/backend/access/spgist/spgxlog.c b/src/backend/access/spgist/spgxlog.c index 01a4e0f252..e016cdb4d3 100644 --- a/src/backend/access/spgist/spgxlog.c +++ b/src/backend/access/spgist/spgxlog.c @@ -1014,9 +1014,7 @@ spg_xlog_startup(void) { opCtx = AllocSetContextCreate(CurrentMemoryContext, "SP-GiST temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } void diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index 0c8c17af33..e9588a7f69 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -1570,10 +1570,8 @@ mXactCachePut(MultiXactId multi, int nmembers, MultiXactMember *members) /* The cache only lives as long as the current transaction */ debug_elog2(DEBUG2, "CachePut: initializing memory context"); MXactContext = AllocSetContextCreate(TopTransactionContext, - "MultiXact Cache Context", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + "MultiXact cache context", + ALLOCSET_SMALL_SIZES); } entry = (mXactCacheEnt *) diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c index 949bfb8b3e..cde0ed300f 100644 --- a/src/backend/access/transam/parallel.c +++ b/src/backend/access/transam/parallel.c @@ -722,10 +722,8 @@ HandleParallelMessages(void) */ if (hpm_context == NULL) /* first time through? */ hpm_context = AllocSetContextCreate(TopMemoryContext, - "HandleParallelMessages context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "HandleParallelMessages", + ALLOCSET_DEFAULT_SIZES); else MemoryContextReset(hpm_context); @@ -962,10 +960,8 @@ ParallelWorkerMain(Datum main_arg) Assert(CurrentResourceOwner == NULL); CurrentResourceOwner = ResourceOwnerCreate(NULL, "parallel toplevel"); CurrentMemoryContext = AllocSetContextCreate(TopMemoryContext, - "parallel worker", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "Parallel worker", + ALLOCSET_DEFAULT_SIZES); /* * Now that we have a resource owner, we can attach to the dynamic shared diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 23f36ead7e..e11b229792 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -1018,9 +1018,7 @@ AtStart_Memory(void) TopTransactionContext = AllocSetContextCreate(TopMemoryContext, "TopTransactionContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * In a top-level transaction, CurTransactionContext is the same as @@ -1078,9 +1076,7 @@ AtSubStart_Memory(void) */ CurTransactionContext = AllocSetContextCreate(CurTransactionContext, "CurTransactionContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); s->curTransactionContext = CurTransactionContext; /* Make the CurTransactionContext active. */ diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f13f9c1fa5..acd95aa740 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4663,9 +4663,7 @@ XLOGShmemInit(void) { walDebugCxt = AllocSetContextCreate(TopMemoryContext, "WAL Debug", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextAllowInCriticalSection(walDebugCxt, true); } #endif diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index c37003a24c..3cd273b19f 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -997,9 +997,7 @@ InitXLogInsert(void) { xloginsert_cxt = AllocSetContextCreate(TopMemoryContext, "WAL record construction", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } if (registered_buffers == NULL) diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index e518e178bb..8feeae05df 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -1069,9 +1069,7 @@ index_register(Oid heap, if (nogc == NULL) nogc = AllocSetContextCreate(NULL, "BootstrapNoGC", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(nogc); diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 8068b82eab..9aa81748ba 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -4747,9 +4747,7 @@ strlist_to_textarray(List *list) memcxt = AllocSetContextCreate(CurrentMemoryContext, "strlist to array", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(memcxt); datums = palloc(sizeof(text *) * list_length(list)); diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 9ac71220a2..c617abb223 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -332,9 +332,7 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, */ anl_context = AllocSetContextCreate(CurrentMemoryContext, "Analyze", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); caller_context = MemoryContextSwitchTo(anl_context); /* @@ -504,9 +502,7 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, col_context = AllocSetContextCreate(anl_context, "Analyze Column", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); old_context = MemoryContextSwitchTo(col_context); for (i = 0; i < attr_cnt; i++) @@ -688,9 +684,7 @@ compute_index_stats(Relation onerel, double totalrows, ind_context = AllocSetContextCreate(anl_context, "Analyze Index", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); old_context = MemoryContextSwitchTo(ind_context); for (ind = 0; ind < nindexes; ind++) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 43bbd90591..dc1f79f594 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -204,9 +204,7 @@ cluster(ClusterStmt *stmt, bool isTopLevel) */ cluster_context = AllocSetContextCreate(PortalContext, "Cluster", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Build the list of relations to cluster. Note that this lives in diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index f45b3304ae..5947e72093 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -1340,9 +1340,7 @@ BeginCopy(bool is_from, */ cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext, "COPY", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcontext = MemoryContextSwitchTo(cstate->copycontext); @@ -1895,9 +1893,7 @@ CopyTo(CopyState cstate) */ cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext, "COPY TO", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); if (cstate->binary) { diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index 50c89b827b..ac4c4ecbe7 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -1018,9 +1018,7 @@ EventTriggerInvoke(List *fn_oid_list, EventTriggerData *trigdata) */ context = AllocSetContextCreate(CurrentMemoryContext, "event trigger context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcontext = MemoryContextSwitchTo(context); /* Call each event trigger. */ @@ -1226,9 +1224,7 @@ EventTriggerBeginCompleteQuery(void) cxt = AllocSetContextCreate(TopMemoryContext, "event trigger state", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); state = MemoryContextAlloc(cxt, sizeof(EventTriggerQueryState)); state->cxt = cxt; slist_init(&(state->SQLDropList)); diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index d14d540b26..85817c6530 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -1903,9 +1903,7 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, */ private_context = AllocSetContextCreate(PortalContext, "ReindexMultipleTables", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_SMALL_SIZES); /* * Define the search keys to find the objects to reindex. For a schema, we diff --git a/src/backend/commands/policy.c b/src/backend/commands/policy.c index bc2e4af82a..d694cf80be 100644 --- a/src/backend/commands/policy.c +++ b/src/backend/commands/policy.c @@ -201,9 +201,7 @@ RelationBuildRowSecurity(Relation relation) */ rscxt = AllocSetContextCreate(CacheMemoryContext, "row security descriptor", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); /* * Since rscxt lives under CacheMemoryContext, it is long-lived. Use a diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 99a659a102..9de22a13d7 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -3339,9 +3339,7 @@ afterTriggerAddEvent(AfterTriggerEventList *events, afterTriggers.event_cxt = AllocSetContextCreate(TopTransactionContext, "AfterTriggerEvents", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Chunk size starts at 1KB and is allowed to increase up to 1MB. @@ -3780,9 +3778,7 @@ afterTriggerInvokeEvents(AfterTriggerEventList *events, per_tuple_context = AllocSetContextCreate(CurrentMemoryContext, "AfterTriggerTupleContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); for_each_chunk(chunk, *events) { diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 0563e63474..58bbf5548b 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -209,9 +209,7 @@ vacuum(int options, RangeVar *relation, Oid relid, VacuumParams *params, */ vac_context = AllocSetContextCreate(PortalContext, "Vacuum", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * If caller didn't give us a buffer strategy object, make one in the diff --git a/src/backend/executor/execUtils.c b/src/backend/executor/execUtils.c index e937cf8e7e..a3bcb100da 100644 --- a/src/backend/executor/execUtils.c +++ b/src/backend/executor/execUtils.c @@ -80,9 +80,7 @@ CreateExecutorState(void) */ qcontext = AllocSetContextCreate(CurrentMemoryContext, "ExecutorState", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Make the EState node within the per-query context. This way, we don't @@ -229,9 +227,7 @@ CreateExprContext(EState *estate) econtext->ecxt_per_tuple_memory = AllocSetContextCreate(estate->es_query_cxt, "ExprContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); econtext->ecxt_param_exec_vals = estate->es_param_exec_vals; econtext->ecxt_param_list_info = estate->es_param_list_info; @@ -300,9 +296,7 @@ CreateStandaloneExprContext(void) econtext->ecxt_per_tuple_memory = AllocSetContextCreate(CurrentMemoryContext, "ExprContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); econtext->ecxt_param_exec_vals = NULL; econtext->ecxt_param_list_info = NULL; diff --git a/src/backend/executor/functions.c b/src/backend/executor/functions.c index e02fba5232..470db5bb4a 100644 --- a/src/backend/executor/functions.c +++ b/src/backend/executor/functions.c @@ -600,9 +600,7 @@ init_sql_fcache(FmgrInfo *finfo, Oid collation, bool lazyEvalOK) */ fcontext = AllocSetContextCreate(finfo->fn_mcxt, "SQL function data", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcontext = MemoryContextSwitchTo(fcontext); diff --git a/src/backend/executor/nodeFunctionscan.c b/src/backend/executor/nodeFunctionscan.c index a03f6e73fd..5a0f324de0 100644 --- a/src/backend/executor/nodeFunctionscan.c +++ b/src/backend/executor/nodeFunctionscan.c @@ -508,9 +508,7 @@ ExecInitFunctionScan(FunctionScan *node, EState *estate, int eflags) */ scanstate->argcontext = AllocSetContextCreate(CurrentMemoryContext, "Table function arguments", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); return scanstate; } diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index 9ed09a7b0c..6375d9bfda 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -344,15 +344,11 @@ ExecHashTableCreate(Hash *node, List *hashOperators, bool keepNulls) */ hashtable->hashCxt = AllocSetContextCreate(CurrentMemoryContext, "HashTableContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); hashtable->batchCxt = AllocSetContextCreate(hashtable->hashCxt, "HashBatchContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* Allocate data that will live for the life of the hashjoin */ diff --git a/src/backend/executor/nodeRecursiveunion.c b/src/backend/executor/nodeRecursiveunion.c index e76405a56e..39be191c45 100644 --- a/src/backend/executor/nodeRecursiveunion.c +++ b/src/backend/executor/nodeRecursiveunion.c @@ -200,15 +200,11 @@ ExecInitRecursiveUnion(RecursiveUnion *node, EState *estate, int eflags) rustate->tempContext = AllocSetContextCreate(CurrentMemoryContext, "RecursiveUnion", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); rustate->tableContext = AllocSetContextCreate(CurrentMemoryContext, "RecursiveUnion hash table", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } /* diff --git a/src/backend/executor/nodeSetOp.c b/src/backend/executor/nodeSetOp.c index 2d81d46927..633580b436 100644 --- a/src/backend/executor/nodeSetOp.c +++ b/src/backend/executor/nodeSetOp.c @@ -507,9 +507,7 @@ ExecInitSetOp(SetOp *node, EState *estate, int eflags) setopstate->tempContext = AllocSetContextCreate(CurrentMemoryContext, "SetOp", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * If hashing, we also need a longer-lived context to store the hash @@ -520,9 +518,7 @@ ExecInitSetOp(SetOp *node, EState *estate, int eflags) setopstate->tableContext = AllocSetContextCreate(CurrentMemoryContext, "SetOp hash table", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Tuple table initialization diff --git a/src/backend/executor/nodeSubplan.c b/src/backend/executor/nodeSubplan.c index e503494edd..2cf169f956 100644 --- a/src/backend/executor/nodeSubplan.c +++ b/src/backend/executor/nodeSubplan.c @@ -776,16 +776,12 @@ ExecInitSubPlan(SubPlan *subplan, PlanState *parent) sstate->hashtablecxt = AllocSetContextCreate(CurrentMemoryContext, "Subplan HashTable Context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* and a small one for the hash tables to use as temp storage */ sstate->hashtempcxt = AllocSetContextCreate(CurrentMemoryContext, "Subplan HashTable Temp Context", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); /* and a short-lived exprcontext for function evaluation */ sstate->innerecontext = CreateExprContext(estate); /* Silly little array of column numbers 1..n */ diff --git a/src/backend/executor/nodeUnique.c b/src/backend/executor/nodeUnique.c index 4caae34b97..f45c79232d 100644 --- a/src/backend/executor/nodeUnique.c +++ b/src/backend/executor/nodeUnique.c @@ -133,9 +133,7 @@ ExecInitUnique(Unique *node, EState *estate, int eflags) uniquestate->tempContext = AllocSetContextCreate(CurrentMemoryContext, "Unique", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Tuple table initialization diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c index d4c88a1f0e..371548ceb3 100644 --- a/src/backend/executor/nodeWindowAgg.c +++ b/src/backend/executor/nodeWindowAgg.c @@ -1801,10 +1801,8 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) /* Create long-lived context for storage of partition-local memory etc */ winstate->partcontext = AllocSetContextCreate(CurrentMemoryContext, - "WindowAgg_Partition", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "WindowAgg Partition", + ALLOCSET_DEFAULT_SIZES); /* * Create mid-lived context for aggregate trans values etc. @@ -1814,10 +1812,8 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) */ winstate->aggcontext = AllocSetContextCreate(CurrentMemoryContext, - "WindowAgg_Aggregates", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "WindowAgg Aggregates", + ALLOCSET_DEFAULT_SIZES); /* * tuple table initialization @@ -2321,10 +2317,8 @@ initialize_peragg(WindowAggState *winstate, WindowFunc *wfunc, if (OidIsValid(invtransfn_oid)) peraggstate->aggcontext = AllocSetContextCreate(CurrentMemoryContext, - "WindowAgg_AggregatePrivate", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "WindowAgg Per Aggregate", + ALLOCSET_DEFAULT_SIZES); else peraggstate->aggcontext = winstate->aggcontext; diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index 38de18006d..38767ae4ce 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -142,14 +142,10 @@ SPI_connect(void) */ _SPI_current->procCxt = AllocSetContextCreate(TopTransactionContext, "SPI Proc", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); _SPI_current->execCxt = AllocSetContextCreate(TopTransactionContext, "SPI Exec", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* ... and switch to procedure's context */ _SPI_current->savedcxt = MemoryContextSwitchTo(_SPI_current->procCxt); @@ -1744,9 +1740,7 @@ spi_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo) tuptabcxt = AllocSetContextCreate(CurrentMemoryContext, "SPI TupTable", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(tuptabcxt); _SPI_current->tuptable = tuptable = (SPITupleTable *) @@ -2615,14 +2609,11 @@ _SPI_make_plan_non_temp(SPIPlanPtr plan) /* * Create a memory context for the plan, underneath the procedure context. - * We don't expect the plan to be very large, so use smaller-than-default - * alloc parameters. + * We don't expect the plan to be very large. */ plancxt = AllocSetContextCreate(parentcxt, "SPI Plan", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); oldcxt = MemoryContextSwitchTo(plancxt); /* Copy the SPI_plan struct and subsidiary data into the new context */ @@ -2689,9 +2680,7 @@ _SPI_save_plan(SPIPlanPtr plan) */ plancxt = AllocSetContextCreate(CurrentMemoryContext, "SPI Plan", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); oldcxt = MemoryContextSwitchTo(plancxt); /* Copy the SPI plan into its own context */ diff --git a/src/backend/executor/tqueue.c b/src/backend/executor/tqueue.c index 58d0eeaf0b..344e623c94 100644 --- a/src/backend/executor/tqueue.c +++ b/src/backend/executor/tqueue.c @@ -281,9 +281,7 @@ tqueueReceiveSlot(TupleTableSlot *slot, DestReceiver *self) tqueue->tmpcontext = AllocSetContextCreate(tqueue->mycontext, "tqueue sender temp context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcontext = MemoryContextSwitchTo(tqueue->tmpcontext); } diff --git a/src/backend/libpq/be-fsstubs.c b/src/backend/libpq/be-fsstubs.c index b64ef8b5c4..764f602aaa 100644 --- a/src/backend/libpq/be-fsstubs.c +++ b/src/backend/libpq/be-fsstubs.c @@ -79,9 +79,7 @@ static MemoryContext fscxt = NULL; if (fscxt == NULL) \ fscxt = AllocSetContextCreate(TopMemoryContext, \ "Filesystem", \ - ALLOCSET_DEFAULT_MINSIZE, \ - ALLOCSET_DEFAULT_INITSIZE, \ - ALLOCSET_DEFAULT_MAXSIZE); \ + ALLOCSET_DEFAULT_SIZES); \ } while (0) diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 1b4bbce42d..d612c11159 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -387,10 +387,8 @@ tokenize_file(const char *filename, FILE *file, MemoryContext oldcxt; linecxt = AllocSetContextCreate(CurrentMemoryContext, - "tokenize file cxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "tokenize_file", + ALLOCSET_SMALL_SIZES); oldcxt = MemoryContextSwitchTo(linecxt); *lines = *line_nums = NIL; @@ -1817,9 +1815,7 @@ load_hba(void) Assert(PostmasterContext); hbacxt = AllocSetContextCreate(PostmasterContext, "hba parser context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_SMALL_SIZES); oldcxt = MemoryContextSwitchTo(hbacxt); forthree(line, hba_lines, line_num, hba_line_nums, raw_line, hba_raw_lines) { @@ -2195,9 +2191,7 @@ load_ident(void) Assert(PostmasterContext); ident_context = AllocSetContextCreate(PostmasterContext, "ident parser context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_SMALL_SIZES); oldcxt = MemoryContextSwitchTo(ident_context); forboth(line_cell, ident_lines, num_cell, ident_line_nums) { diff --git a/src/backend/optimizer/geqo/geqo_eval.c b/src/backend/optimizer/geqo/geqo_eval.c index 88acebc1f2..fb2ab77422 100644 --- a/src/backend/optimizer/geqo/geqo_eval.c +++ b/src/backend/optimizer/geqo/geqo_eval.c @@ -74,9 +74,7 @@ geqo_eval(PlannerInfo *root, Gene *tour, int num_gene) */ mycontext = AllocSetContextCreate(CurrentMemoryContext, "GEQO", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(mycontext); /* diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 4496fde056..e1baf71e38 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -4378,9 +4378,7 @@ inline_function(Oid funcid, Oid result_type, Oid result_collid, */ mycxt = AllocSetContextCreate(CurrentMemoryContext, "inline_function", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(mycxt); /* Fetch the function body */ @@ -4896,9 +4894,7 @@ inline_set_returning_function(PlannerInfo *root, RangeTblEntry *rte) */ mycxt = AllocSetContextCreate(CurrentMemoryContext, "inline_set_returning_function", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(mycxt); /* diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 3768f50bcf..1a92ca1deb 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -462,9 +462,7 @@ AutoVacLauncherMain(int argc, char *argv[]) */ AutovacMemCxt = AllocSetContextCreate(TopMemoryContext, "Autovacuum Launcher", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(AutovacMemCxt); /* @@ -894,14 +892,10 @@ rebuild_database_list(Oid newdb) newcxt = AllocSetContextCreate(AutovacMemCxt, "AV dblist", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); tmpcxt = AllocSetContextCreate(newcxt, "tmp AV dblist", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(tmpcxt); /* @@ -1111,9 +1105,7 @@ do_start_worker(void) */ tmpcxt = AllocSetContextCreate(CurrentMemoryContext, "Start worker tmp cxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(tmpcxt); /* use fresh stats */ @@ -1911,9 +1903,7 @@ do_autovacuum(void) */ AutovacMemCxt = AllocSetContextCreate(TopMemoryContext, "AV worker", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(AutovacMemCxt); /* @@ -2183,9 +2173,7 @@ do_autovacuum(void) */ PortalContext = AllocSetContextCreate(AutovacMemCxt, "Autovacuum Portal", - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Perform operations on collected tables. diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c index 00f03d8acb..10020349a2 100644 --- a/src/backend/postmaster/bgwriter.c +++ b/src/backend/postmaster/bgwriter.c @@ -160,9 +160,7 @@ BackgroundWriterMain(void) */ bgwriter_context = AllocSetContextCreate(TopMemoryContext, "Background Writer", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(bgwriter_context); WritebackContextInit(&wb_context, &bgwriter_flush_after); diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index 8d4b3539b1..d702a4864d 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -245,9 +245,7 @@ CheckpointerMain(void) */ checkpointer_context = AllocSetContextCreate(TopMemoryContext, "Checkpointer", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(checkpointer_context); /* diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 8fa9edbf72..2f99aea791 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -4792,9 +4792,7 @@ pgstat_setup_memcxt(void) if (!pgStatLocalContext) pgStatLocalContext = AllocSetContextCreate(TopMemoryContext, "Statistics snapshot", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); } diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 05f3f14e35..a28e215e2d 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -583,9 +583,7 @@ PostmasterMain(int argc, char *argv[]) */ PostmasterContext = AllocSetContextCreate(TopMemoryContext, "Postmaster", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(PostmasterContext); /* Initialize paths to installation files */ diff --git a/src/backend/postmaster/walwriter.c b/src/backend/postmaster/walwriter.c index 228190a836..11ec56aebb 100644 --- a/src/backend/postmaster/walwriter.c +++ b/src/backend/postmaster/walwriter.c @@ -142,9 +142,7 @@ WalWriterMain(void) */ walwriter_context = AllocSetContextCreate(TopMemoryContext, "Wal Writer", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(walwriter_context); /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index ecf9a03318..1512be5322 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -127,10 +127,8 @@ StartupDecodingContext(List *output_plugin_options, slot = MyReplicationSlot; context = AllocSetContextCreate(CurrentMemoryContext, - "Logical Decoding Context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "Logical decoding context", + ALLOCSET_DEFAULT_SIZES); old_context = MemoryContextSwitchTo(context); ctx = palloc0(sizeof(LogicalDecodingContext)); diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 213ce34674..43b584cf7e 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -232,9 +232,7 @@ ReorderBufferAllocate(void) /* allocate memory in own context, to have better accountability */ new_ctx = AllocSetContextCreate(CurrentMemoryContext, "ReorderBuffer", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); buffer = (ReorderBuffer *) MemoryContextAlloc(new_ctx, sizeof(ReorderBuffer)); @@ -2317,7 +2315,7 @@ ReorderBufferSerializeChange(ReorderBuffer *rb, ReorderBufferTXN *txn, if (write(fd, rb->outbuf, ondisk->size) != ondisk->size) { - int save_errno = errno; + int save_errno = errno; CloseTransientFile(fd); errno = save_errno; diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index b5fa3dbbc0..8b59fc5a16 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -289,9 +289,7 @@ AllocateSnapshotBuilder(ReorderBuffer *reorder, /* allocate memory in own context, to have better accountability */ context = AllocSetContextCreate(CurrentMemoryContext, "snapshot builder context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcontext = MemoryContextSwitchTo(context); builder = palloc0(sizeof(SnapBuild)); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index a0dba194a6..1ea2a5cfdf 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1309,9 +1309,7 @@ exec_replication_command(const char *cmd_string) cmd_context = AllocSetContextCreate(CurrentMemoryContext, "Replication command context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); old_context = MemoryContextSwitchTo(cmd_context); replication_scanner_init(cmd_string); diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c index 53981794b9..ca2388789d 100644 --- a/src/backend/storage/buffer/localbuf.c +++ b/src/backend/storage/buffer/localbuf.c @@ -511,9 +511,7 @@ GetLocalBufferStorage(void) LocalBufferContext = AllocSetContextCreate(TopMemoryContext, "LocalBufferContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* Start with a 16-buffer request; subsequent ones double each time */ num_bufs = Max(num_bufs_in_block * 2, 16); diff --git a/src/backend/storage/file/reinit.c b/src/backend/storage/file/reinit.c index 7e8138b42a..6b98131e54 100644 --- a/src/backend/storage/file/reinit.c +++ b/src/backend/storage/file/reinit.c @@ -65,9 +65,7 @@ ResetUnloggedRelations(int op) */ tmpctx = AllocSetContextCreate(CurrentMemoryContext, "ResetUnloggedRelations", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldctx = MemoryContextSwitchTo(tmpctx); /* diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 303e99c65b..53b45d72fe 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -285,9 +285,7 @@ init_lwlock_stats(void) */ lwlock_stats_cxt = AllocSetContextCreate(TopMemoryContext, "LWLock stats", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextAllowInCriticalSection(lwlock_stats_cxt, true); MemSet(&ctl, 0, sizeof(ctl)); diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index f329d1538c..1287142918 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -208,9 +208,7 @@ mdinit(void) { MdCxt = AllocSetContextCreate(TopMemoryContext, "MdSmgr", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Create pending-operations hashtable if we need it. Currently, we need @@ -231,10 +229,8 @@ mdinit(void) * practice. */ pendingOpsCxt = AllocSetContextCreate(MdCxt, - "Pending Ops Context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "Pending ops context", + ALLOCSET_DEFAULT_SIZES); MemoryContextAllowInCriticalSection(pendingOpsCxt, true); MemSet(&hash_ctl, 0, sizeof(hash_ctl)); diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index b185c1b5eb..98ccbbb4d1 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -1253,9 +1253,7 @@ exec_parse_message(const char *query_string, /* string to execute */ unnamed_stmt_context = AllocSetContextCreate(MessageContext, "unnamed prepared statement", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcontext = MemoryContextSwitchTo(unnamed_stmt_context); } @@ -3794,9 +3792,7 @@ PostgresMain(int argc, char *argv[], */ MessageContext = AllocSetContextCreate(TopMemoryContext, "MessageContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Remember stand-alone backend startup time diff --git a/src/backend/tsearch/spell.c b/src/backend/tsearch/spell.c index 821f611e4f..9c7ba85eb5 100644 --- a/src/backend/tsearch/spell.c +++ b/src/backend/tsearch/spell.c @@ -92,9 +92,7 @@ NIStartBuild(IspellDict *Conf) */ Conf->buildCxt = AllocSetContextCreate(CurTransactionContext, "Ispell dictionary init context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } /* diff --git a/src/backend/utils/adt/array_expanded.c b/src/backend/utils/adt/array_expanded.c index 7dd7e3fbcb..94eb19d45d 100644 --- a/src/backend/utils/adt/array_expanded.c +++ b/src/backend/utils/adt/array_expanded.c @@ -63,9 +63,7 @@ expand_array(Datum arraydatum, MemoryContext parentcontext, */ objcxt = AllocSetContextCreate(parentcontext, "expanded array", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_START_SMALL_SIZES); /* Set up expanded array header */ eah = (ExpandedArrayHeader *) diff --git a/src/backend/utils/adt/arrayfuncs.c b/src/backend/utils/adt/arrayfuncs.c index 8fbd850146..1db7bf0a35 100644 --- a/src/backend/utils/adt/arrayfuncs.c +++ b/src/backend/utils/adt/arrayfuncs.c @@ -4957,9 +4957,7 @@ initArrayResult(Oid element_type, MemoryContext rcontext, bool subcontext) if (subcontext) arr_context = AllocSetContextCreate(rcontext, "accumArrayResult", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); astate = (ArrayBuildState *) MemoryContextAlloc(arr_context, sizeof(ArrayBuildState)); @@ -5161,9 +5159,7 @@ initArrayResultArr(Oid array_type, Oid element_type, MemoryContext rcontext, if (subcontext) arr_context = AllocSetContextCreate(rcontext, "accumArrayResultArr", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* Note we initialize all fields to zero */ astate = (ArrayBuildStateArr *) diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c index a80a20ecee..996007d483 100644 --- a/src/backend/utils/adt/jsonfuncs.c +++ b/src/backend/utils/adt/jsonfuncs.c @@ -1503,9 +1503,7 @@ each_worker_jsonb(FunctionCallInfo fcinfo, const char *funcname, bool as_text) tmp_cxt = AllocSetContextCreate(CurrentMemoryContext, "jsonb_each temporary cxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); it = JsonbIteratorInit(&jb->root); @@ -1641,9 +1639,7 @@ each_worker(FunctionCallInfo fcinfo, bool as_text) state->lex = lex; state->tmp_cxt = AllocSetContextCreate(CurrentMemoryContext, "json_each temporary cxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); pg_parse_json(lex, sem); @@ -1822,9 +1818,7 @@ elements_worker_jsonb(FunctionCallInfo fcinfo, const char *funcname, tmp_cxt = AllocSetContextCreate(CurrentMemoryContext, "jsonb_array_elements temporary cxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); it = JsonbIteratorInit(&jb->root); @@ -1962,9 +1956,7 @@ elements_worker(FunctionCallInfo fcinfo, const char *funcname, bool as_text) state->lex = lex; state->tmp_cxt = AllocSetContextCreate(CurrentMemoryContext, "json_array_elements temporary cxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); pg_parse_json(lex, sem); diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c index 7ed5bcb93d..b144920ec6 100644 --- a/src/backend/utils/adt/xml.c +++ b/src/backend/utils/adt/xml.c @@ -1455,10 +1455,8 @@ xml_memory_init(void) /* Create memory context if not there already */ if (LibxmlContext == NULL) LibxmlContext = AllocSetContextCreate(TopMemoryContext, - "LibxmlContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "Libxml context", + ALLOCSET_DEFAULT_SIZES); /* Re-establish the callbacks even if already set */ xmlMemSetup(xml_pfree, xml_palloc, xml_repalloc, xml_pstrdup); diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c index e929616c97..db7099fc0e 100644 --- a/src/backend/utils/cache/catcache.c +++ b/src/backend/utils/cache/catcache.c @@ -536,9 +536,7 @@ CreateCacheMemoryContext(void) if (!CacheMemoryContext) CacheMemoryContext = AllocSetContextCreate(TopMemoryContext, "CacheMemoryContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } diff --git a/src/backend/utils/cache/evtcache.c b/src/backend/utils/cache/evtcache.c index 6fc1df880b..8a620a51c8 100644 --- a/src/backend/utils/cache/evtcache.c +++ b/src/backend/utils/cache/evtcache.c @@ -105,9 +105,7 @@ BuildEventTriggerCache(void) EventTriggerCacheContext = AllocSetContextCreate(CacheMemoryContext, "EventTriggerCache", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); CacheRegisterSyscacheCallback(EVENTTRIGGEROID, InvalidateEventCacheCallback, (Datum) 0); diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index f42a62d500..c96a86500a 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -159,15 +159,13 @@ CreateCachedPlan(Node *raw_parse_tree, /* * Make a dedicated memory context for the CachedPlanSource and its * permanent subsidiary data. It's probably not going to be large, but - * just in case, use the default maxsize parameter. Initially it's a - * child of the caller's context (which we assume to be transient), so - * that it will be cleaned up on error. + * just in case, allow it to grow large. Initially it's a child of the + * caller's context (which we assume to be transient), so that it will be + * cleaned up on error. */ source_context = AllocSetContextCreate(CurrentMemoryContext, "CachedPlanSource", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_START_SMALL_SIZES); /* * Create and fill the CachedPlanSource struct within the new context. @@ -359,9 +357,7 @@ CompleteCachedPlan(CachedPlanSource *plansource, /* Again, it's a good bet the querytree_context can be small */ querytree_context = AllocSetContextCreate(source_context, "CachedPlanQuery", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_START_SMALL_SIZES); MemoryContextSwitchTo(querytree_context); querytree_list = (List *) copyObject(querytree_list); } @@ -733,9 +729,7 @@ RevalidateCachedQuery(CachedPlanSource *plansource) */ querytree_context = AllocSetContextCreate(CurrentMemoryContext, "CachedPlanQuery", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_START_SMALL_SIZES); oldcxt = MemoryContextSwitchTo(querytree_context); qlist = (List *) copyObject(tlist); @@ -955,17 +949,14 @@ BuildCachedPlan(CachedPlanSource *plansource, List *qlist, /* * Normally we make a dedicated memory context for the CachedPlan and its * subsidiary data. (It's probably not going to be large, but just in - * case, use the default maxsize parameter. It's transient for the - * moment.) But for a one-shot plan, we just leave it in the caller's - * memory context. + * case, allow it to grow large. It's transient for the moment.) But for + * a one-shot plan, we just leave it in the caller's memory context. */ if (!plansource->is_oneshot) { plan_context = AllocSetContextCreate(CurrentMemoryContext, "CachedPlan", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_START_SMALL_SIZES); /* * Copy plan into the new context. @@ -1351,9 +1342,7 @@ CopyCachedPlan(CachedPlanSource *plansource) source_context = AllocSetContextCreate(CurrentMemoryContext, "CachedPlanSource", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_START_SMALL_SIZES); oldcxt = MemoryContextSwitchTo(source_context); @@ -1384,9 +1373,7 @@ CopyCachedPlan(CachedPlanSource *plansource) querytree_context = AllocSetContextCreate(source_context, "CachedPlanQuery", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_START_SMALL_SIZES); MemoryContextSwitchTo(querytree_context); newsource->query_list = (List *) copyObject(plansource->query_list); newsource->relationOids = (List *) copyObject(plansource->relationOids); diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 8d2ad018bb..79e0b1ff48 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -659,14 +659,11 @@ RelationBuildRuleLock(Relation relation) int maxlocks; /* - * Make the private context. Parameters are set on the assumption that - * it'll probably not contain much data. + * Make the private context. Assume it'll not contain much data. */ rulescxt = AllocSetContextCreate(CacheMemoryContext, RelationGetRelationName(relation), - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); relation->rd_rulescxt = rulescxt; /* @@ -1248,15 +1245,10 @@ RelationInitIndexAccessInfo(Relation relation) * Make the private context to hold index access info. The reason we need * a context, and not just a couple of pallocs, is so that we won't leak * any subsidiary info attached to fmgr lookup records. - * - * Context parameters are set on the assumption that it'll probably not - * contain much data. */ indexcxt = AllocSetContextCreate(CacheMemoryContext, RelationGetRelationName(relation), - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); relation->rd_indexcxt = indexcxt; /* @@ -4948,9 +4940,7 @@ load_relcache_init_file(bool shared) */ indexcxt = AllocSetContextCreate(CacheMemoryContext, RelationGetRelationName(rel), - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); rel->rd_indexcxt = indexcxt; /* diff --git a/src/backend/utils/cache/ts_cache.c b/src/backend/utils/cache/ts_cache.c index 5e4de431dd..50f17438fb 100644 --- a/src/backend/utils/cache/ts_cache.c +++ b/src/backend/utils/cache/ts_cache.c @@ -295,9 +295,7 @@ lookup_ts_dictionary_cache(Oid dictId) /* Create private memory context the first time through */ saveCtx = AllocSetContextCreate(CacheMemoryContext, NameStr(dict->dictname), - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); } else { diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c index ea6f787a52..9150fe832f 100644 --- a/src/backend/utils/cache/typcache.c +++ b/src/backend/utils/cache/typcache.c @@ -756,9 +756,7 @@ load_domaintype_info(TypeCacheEntry *typentry) cxt = AllocSetContextCreate(CurrentMemoryContext, "Domain constraints", - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); dcc = (DomainConstraintCache *) MemoryContextAlloc(cxt, sizeof(DomainConstraintCache)); dcc->constraints = NIL; @@ -841,9 +839,7 @@ load_domaintype_info(TypeCacheEntry *typentry) cxt = AllocSetContextCreate(CurrentMemoryContext, "Domain constraints", - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); dcc = (DomainConstraintCache *) MemoryContextAlloc(cxt, sizeof(DomainConstraintCache)); dcc->constraints = NIL; diff --git a/src/backend/utils/fmgr/funcapi.c b/src/backend/utils/fmgr/funcapi.c index 5d179ae8a8..5d49fe5b50 100644 --- a/src/backend/utils/fmgr/funcapi.c +++ b/src/backend/utils/fmgr/funcapi.c @@ -73,9 +73,7 @@ init_MultiFuncCall(PG_FUNCTION_ARGS) */ multi_call_ctx = AllocSetContextCreate(fcinfo->flinfo->fn_mcxt, "SRF multi-call context", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); /* * Allocate suitably long-lived space and zero it diff --git a/src/backend/utils/hash/dynahash.c b/src/backend/utils/hash/dynahash.c index d35052aea6..bb835ba946 100644 --- a/src/backend/utils/hash/dynahash.c +++ b/src/backend/utils/hash/dynahash.c @@ -327,9 +327,7 @@ hash_create(const char *tabname, long nelem, HASHCTL *info, int flags) CurrentDynaHashCxt = TopMemoryContext; CurrentDynaHashCxt = AllocSetContextCreate(CurrentDynaHashCxt, tabname, - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } /* Initialize the hash header, plus a copy of the table name */ diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index d17197267e..824d5abf11 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -201,9 +201,7 @@ PerformAuthentication(Port *port) if (PostmasterContext == NULL) PostmasterContext = AllocSetContextCreate(TopMemoryContext, "Postmaster", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); if (!load_hba()) { diff --git a/src/backend/utils/misc/guc-file.l b/src/backend/utils/misc/guc-file.l index 48052f9320..dae5015a32 100644 --- a/src/backend/utils/misc/guc-file.l +++ b/src/backend/utils/misc/guc-file.l @@ -145,9 +145,7 @@ ProcessConfigFile(GucContext context) */ config_cxt = AllocSetContextCreate(CurrentMemoryContext, "config file processing", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); caller_cxt = MemoryContextSwitchTo(config_cxt); /* diff --git a/src/backend/utils/misc/tzparser.c b/src/backend/utils/misc/tzparser.c index a960343baa..a053e22439 100644 --- a/src/backend/utils/misc/tzparser.c +++ b/src/backend/utils/misc/tzparser.c @@ -450,9 +450,7 @@ load_tzoffsets(const char *filename) */ tmpContext = AllocSetContextCreate(CurrentMemoryContext, "TZParserMemory", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); oldContext = MemoryContextSwitchTo(tmpContext); /* Initialize array at a reasonable size */ diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c index d26991ed23..43c85234ce 100644 --- a/src/backend/utils/mmgr/aset.c +++ b/src/backend/utils/mmgr/aset.c @@ -427,10 +427,14 @@ randomize_mem(char *ptr, size_t size) * Create a new AllocSet context. * * parent: parent context, or NULL if top-level context - * name: name of context (for debugging --- string will be copied) + * name: name of context (for debugging only, need not be unique) * minContextSize: minimum context size * initBlockSize: initial allocation block size * maxBlockSize: maximum allocation block size + * + * Notes: the name string will be copied into context-lifespan storage. + * Most callers should abstract the context size parameters using a macro + * such as ALLOCSET_DEFAULT_SIZES. */ MemoryContext AllocSetContextCreate(MemoryContext parent, diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index 6b7894213c..5cf388f9d6 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -91,16 +91,13 @@ MemoryContextInit(void) AssertState(TopMemoryContext == NULL); /* - * Initialize TopMemoryContext as an AllocSetContext with slow growth rate - * --- we don't really expect much to be allocated in it. - * - * (There is special-case code in MemoryContextCreate() for this call.) + * First, initialize TopMemoryContext, which will hold the MemoryContext + * nodes for all other contexts. (There is special-case code in + * MemoryContextCreate() to handle this call.) */ TopMemoryContext = AllocSetContextCreate((MemoryContext) NULL, "TopMemoryContext", - 0, - 8 * 1024, - 8 * 1024); + ALLOCSET_DEFAULT_SIZES); /* * Not having any other place to point CurrentMemoryContext, make it point diff --git a/src/backend/utils/mmgr/portalmem.c b/src/backend/utils/mmgr/portalmem.c index 425cae12ea..8286800380 100644 --- a/src/backend/utils/mmgr/portalmem.c +++ b/src/backend/utils/mmgr/portalmem.c @@ -108,9 +108,7 @@ EnablePortalManager(void) PortalMemory = AllocSetContextCreate(TopMemoryContext, "PortalMemory", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); ctl.keysize = MAX_PORTALNAME_LEN; ctl.entrysize = sizeof(PortalHashEnt); @@ -221,9 +219,7 @@ CreatePortal(const char *name, bool allowDup, bool dupSilent) /* initialize portal heap context; typically it won't store much */ portal->heap = AllocSetContextCreate(PortalMemory, "PortalHeapMemory", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); /* create a resource owner for the portal */ portal->resowner = ResourceOwnerCreate(CurTransactionResourceOwner, @@ -361,9 +357,7 @@ PortalCreateHoldStore(Portal portal) portal->holdContext = AllocSetContextCreate(PortalMemory, "PortalHoldContext", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Create the tuple store, selecting cross-transaction temp files, and diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index ae384a8546..c8fbcf8fcc 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -654,9 +654,7 @@ tuplesort_begin_common(int workMem, bool randomAccess) */ sortcontext = AllocSetContextCreate(CurrentMemoryContext, "TupleSort main", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Caller tuple (e.g. IndexTuple) memory context. @@ -669,9 +667,7 @@ tuplesort_begin_common(int workMem, bool randomAccess) */ tuplecontext = AllocSetContextCreate(sortcontext, "Caller tuples", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); /* * Make the Tuplesortstate within the per-sort context. This way, we diff --git a/src/include/utils/memutils.h b/src/include/utils/memutils.h index ae07705b6b..e6334a2038 100644 --- a/src/include/utils/memutils.h +++ b/src/include/utils/memutils.h @@ -142,14 +142,26 @@ extern MemoryContext AllocSetContextCreate(MemoryContext parent, #define ALLOCSET_DEFAULT_MINSIZE 0 #define ALLOCSET_DEFAULT_INITSIZE (8 * 1024) #define ALLOCSET_DEFAULT_MAXSIZE (8 * 1024 * 1024) +#define ALLOCSET_DEFAULT_SIZES \ + ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE /* - * Recommended alloc parameters for "small" contexts that are not expected + * Recommended alloc parameters for "small" contexts that are never expected * to contain much data (for example, a context to contain a query plan). */ #define ALLOCSET_SMALL_MINSIZE 0 #define ALLOCSET_SMALL_INITSIZE (1 * 1024) #define ALLOCSET_SMALL_MAXSIZE (8 * 1024) +#define ALLOCSET_SMALL_SIZES \ + ALLOCSET_SMALL_MINSIZE, ALLOCSET_SMALL_INITSIZE, ALLOCSET_SMALL_MAXSIZE + +/* + * Recommended alloc parameters for contexts that should start out small, + * but might sometimes grow big. + */ +#define ALLOCSET_START_SMALL_SIZES \ + ALLOCSET_SMALL_MINSIZE, ALLOCSET_SMALL_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE + /* * Threshold above which a request in an AllocSet context is certain to be diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 82bde6e442..2cd761496d 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -3205,9 +3205,7 @@ plperl_return_next(SV *sv) current_call_data->tmp_cxt = AllocSetContextCreate(CurrentMemoryContext, "PL/Perl return_next temporary cxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } old_cxt = MemoryContextSwitchTo(current_call_data->tmp_cxt); @@ -3460,9 +3458,7 @@ plperl_spi_prepare(char *query, int argc, SV **argv) ************************************************************/ plan_cxt = AllocSetContextCreate(TopMemoryContext, "PL/Perl spi_prepare query", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); MemoryContextSwitchTo(plan_cxt); qdesc = (plperl_query_desc *) palloc0(sizeof(plperl_query_desc)); snprintf(qdesc->qname, sizeof(qdesc->qname), "%p", qdesc); @@ -3479,9 +3475,7 @@ plperl_spi_prepare(char *query, int argc, SV **argv) ************************************************************/ work_cxt = AllocSetContextCreate(CurrentMemoryContext, "PL/Perl spi_prepare workspace", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(work_cxt); /************************************************************ diff --git a/src/pl/plpgsql/src/pl_comp.c b/src/pl/plpgsql/src/pl_comp.c index 38aa030303..4ceb402c92 100644 --- a/src/pl/plpgsql/src/pl_comp.c +++ b/src/pl/plpgsql/src/pl_comp.c @@ -340,9 +340,7 @@ do_compile(FunctionCallInfo fcinfo, */ func_cxt = AllocSetContextCreate(TopMemoryContext, "PL/pgSQL function context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); plpgsql_compile_tmp_cxt = MemoryContextSwitchTo(func_cxt); function->fn_signature = format_procedure(fcinfo->flinfo->fn_oid); @@ -829,10 +827,8 @@ plpgsql_compile_inline(char *proc_source) * its own memory context, so it can be reclaimed easily. */ func_cxt = AllocSetContextCreate(CurrentMemoryContext, - "PL/pgSQL function context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + "PL/pgSQL inline code context", + ALLOCSET_DEFAULT_SIZES); plpgsql_compile_tmp_cxt = MemoryContextSwitchTo(func_cxt); function->fn_signature = pstrdup(func_name); diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index f9b3b22d08..2f8b6ff2f2 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -1092,9 +1092,7 @@ get_stmt_mcontext(PLpgSQL_execstate *estate) estate->stmt_mcontext = AllocSetContextCreate(estate->stmt_mcontext_parent, "PLpgSQL per-statement data", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); } return estate->stmt_mcontext; } @@ -3479,9 +3477,7 @@ plpgsql_estate_setup(PLpgSQL_execstate *estate, { shared_cast_context = AllocSetContextCreate(TopMemoryContext, "PLpgSQL cast info", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(plpgsql_CastHashKey); ctl.entrysize = sizeof(plpgsql_CastHashEntry); diff --git a/src/pl/plpython/plpy_cursorobject.c b/src/pl/plpython/plpy_cursorobject.c index 44ba76e765..0e17a03ce7 100644 --- a/src/pl/plpython/plpy_cursorobject.c +++ b/src/pl/plpython/plpy_cursorobject.c @@ -116,9 +116,7 @@ PLy_cursor_query(const char *query) cursor->closed = false; cursor->mcxt = AllocSetContextCreate(TopMemoryContext, "PL/Python cursor context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); PLy_typeinfo_init(&cursor->result, cursor->mcxt); oldcontext = CurrentMemoryContext; @@ -210,9 +208,7 @@ PLy_cursor_plan(PyObject *ob, PyObject *args) cursor->closed = false; cursor->mcxt = AllocSetContextCreate(TopMemoryContext, "PL/Python cursor context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); PLy_typeinfo_init(&cursor->result, cursor->mcxt); oldcontext = CurrentMemoryContext; diff --git a/src/pl/plpython/plpy_main.c b/src/pl/plpython/plpy_main.c index f95039406a..860b804e54 100644 --- a/src/pl/plpython/plpy_main.c +++ b/src/pl/plpython/plpy_main.c @@ -315,9 +315,7 @@ plpython_inline_handler(PG_FUNCTION_ARGS) MemSet(&proc, 0, sizeof(PLyProcedure)); proc.mcxt = AllocSetContextCreate(TopMemoryContext, "__plpython_inline_block", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); proc.pyname = MemoryContextStrdup(proc.mcxt, "__plpython_inline_block"); proc.langid = codeblock->langOid; proc.result.out.d.typoid = VOIDOID; @@ -416,9 +414,7 @@ PLy_get_scratch_context(PLyExecutionContext *context) context->scratch_ctx = AllocSetContextCreate(TopTransactionContext, "PL/Python scratch context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); return context->scratch_ctx; } diff --git a/src/pl/plpython/plpy_procedure.c b/src/pl/plpython/plpy_procedure.c index 70b75f5d95..2b249b029d 100644 --- a/src/pl/plpython/plpy_procedure.c +++ b/src/pl/plpython/plpy_procedure.c @@ -167,9 +167,7 @@ PLy_procedure_create(HeapTuple procTup, Oid fn_oid, bool is_trigger) cxt = AllocSetContextCreate(TopMemoryContext, procName, - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcxt = MemoryContextSwitchTo(cxt); diff --git a/src/pl/plpython/plpy_spi.c b/src/pl/plpython/plpy_spi.c index 1e965cf85f..09ee06d9e8 100644 --- a/src/pl/plpython/plpy_spi.c +++ b/src/pl/plpython/plpy_spi.c @@ -66,9 +66,7 @@ PLy_spi_prepare(PyObject *self, PyObject *args) plan->mcxt = AllocSetContextCreate(TopMemoryContext, "PL/Python plan context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); oldcontext = MemoryContextSwitchTo(plan->mcxt); nargs = list ? PySequence_Length(list) : 0; @@ -413,9 +411,7 @@ PLy_spi_execute_fetch_result(SPITupleTable *tuptable, uint64 rows, int status) cxt = AllocSetContextCreate(CurrentMemoryContext, "PL/Python temp context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); PLy_typeinfo_init(&args, cxt); oldcontext = CurrentMemoryContext; diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c index 7ad7a4400a..70f2e6d20f 100644 --- a/src/pl/plpython/plpy_typeio.c +++ b/src/pl/plpython/plpy_typeio.c @@ -756,9 +756,7 @@ PLyObject_ToComposite(PLyObToDatum *arg, int32 typmod, PyObject *plrv) /* Create a dummy PLyTypeInfo */ cxt = AllocSetContextCreate(CurrentMemoryContext, "PL/Python temp context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemSet(&info, 0, sizeof(PLyTypeInfo)); PLy_typeinfo_init(&info, cxt); /* Mark it as needing output routines lookup */ @@ -923,9 +921,7 @@ PLyString_ToComposite(PLyTypeInfo *info, TupleDesc desc, PyObject *string) /* Create a dummy PLyTypeInfo */ cxt = AllocSetContextCreate(CurrentMemoryContext, "PL/Python temp context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); MemSet(&locinfo, 0, sizeof(PLyTypeInfo)); PLy_typeinfo_init(&locinfo, cxt); diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 6ee4153ae6..2a335aa219 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -2331,9 +2331,7 @@ pltcl_SPI_prepare(ClientData cdata, Tcl_Interp *interp, ************************************************************/ plan_cxt = AllocSetContextCreate(TopMemoryContext, "PL/TCL spi_prepare query", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_SMALL_MAXSIZE); + ALLOCSET_SMALL_SIZES); MemoryContextSwitchTo(plan_cxt); qdesc = (pltcl_query_desc *) palloc0(sizeof(pltcl_query_desc)); snprintf(qdesc->qname, sizeof(qdesc->qname), "%p", qdesc); From a6f0dc701b2f84839761783e87c49d43cd3e31df Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 28 Aug 2016 12:37:23 -0400 Subject: [PATCH 077/871] Update 9.6 release notes through today. --- doc/src/sgml/config.sgml | 24 +++++------- doc/src/sgml/release-9.6.sgml | 71 ++++++++++++++++++++++++++++++++--- 2 files changed, 75 insertions(+), 20 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 4db9c81e56..7c483c6ef3 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -5388,9 +5388,10 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; Process Title - These settings control how the process title as seen - by ps is modified. See - for details. + These settings control how process titles of server processes are + modified. Process titles are typically viewed using programs like + ps or, on Windows, Process Explorer. + See for details. @@ -5403,18 +5404,14 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; Sets the cluster name that appears in the process title for all - processes in this cluster. The name can be any string of less than - NAMEDATALEN characters (64 characters in a standard + server processes in this cluster. The name can be any string of less + than NAMEDATALEN characters (64 characters in a standard build). Only printable ASCII characters may be used in the cluster_name value. Other characters will be replaced with question marks (?). No name is shown if this parameter is set to the empty string '' (which is the default). This parameter can only be set at server start. - - The process title is typically viewed using programs like - ps or, on Windows, Process Explorer. - @@ -5427,11 +5424,10 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; Enables updating of the process title every time a new SQL command - is received by the server. The process title is typically viewed - by the ps command, - or in Windows by using the Process Explorer. - This value defaults to off on Windows platforms due to the - platform's significant overhead for updating the process title. + is received by the server. + This setting defaults to on on most platforms, but it + defaults to off on Windows due to that platform's larger + overhead for updating the process title. Only superusers can change this setting. diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 8d7356e27f..6ec7583485 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -7,7 +7,7 @@ Release Date 2016-??-?? - Current as of 2016-08-08 (commit 34927b292) + Current as of 2016-08-27 (commit b9fe6cbc8) @@ -348,6 +348,7 @@ This commit is also listed under libpq and psql 2016-04-27 [59eb55127] Fix EXPLAIN VERBOSE output for parallel aggregate. 2016-06-09 [c9ce4a1c6] Eliminate "parallel degree" terminology. 2016-06-16 [75be66464] Invent min_parallel_relation_size GUC to replace a hard- +2016-08-16 [f85b1a841] Disable parallel query by default. --> Parallel queries (Robert Haas, Amit Kapila, David Rowley, @@ -365,9 +366,11 @@ This commit is also listed under libpq and psql - Use of parallel query execution can be controlled - through the new configuration parameters , + Parallel query execution is not (yet) enabled by default. + To allow it, set the new configuration + parameter to a + value larger than zero. Additional control over use of parallelism + is available through other new configuration parameters , , , and + + + + Disable by default on + Windows (Takayuki Tsunakawa) + + + + The overhead of updating the process title is much larger on Windows + than most other platforms, and it is also less useful to do it since + most Windows users do not have tools that can display process titles. + + + @@ -2431,6 +2450,22 @@ XXX this is pending backpatch, may need to remove + + Add a nonlocalized version of the severity field in error and notice + messages (Tom Lane) + + + + This change allows client code to determine severity of an error or + notice without having to worry about localized variants of the + severity strings. + + + + + @@ -2957,6 +2992,25 @@ This commit is also listed under libpq and PL/pgSQL + + Add macros to make AllocSetContextCreate() calls simpler + and safer (Tom Lane) + + + + Writing out the individual sizing parameters for a memory context + is now deprecated in favor of using one of the new + macros ALLOCSET_DEFAULT_SIZES, + ALLOCSET_SMALL_SIZES, + or ALLOCSET_START_SMALL_SIZES. + Existing code continues to work, however. + + + + + @@ -3038,19 +3092,24 @@ This commit is also listed under libpq and PL/pgSQL Restructure index access method API to hide most of - it at the C level (Alexander Korotkov) + it at the C level (Alexander Korotkov, Andrew Gierth) This change modernizes the index AM API to look more like the designs we have adopted for foreign data wrappers and tablesample handlers. This simplifies the C code - and should make it more feasible to define index access methods in + and makes it much more practical to define index access methods in installable extensions. A consequence is that most of the columns of the pg_am system catalog have disappeared. + New inspection + functions have been added to allow SQL queries to determine + index AM properties that used to be discoverable + from pg_am. From 39d866433e6fb1c385eee8dc67843097b8703add Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 28 Aug 2016 17:40:06 -0400 Subject: [PATCH 078/871] Make another editorial pass over the 9.6 release notes. I think they're pretty much release-quality now. --- doc/src/sgml/func.sgml | 28 +-- doc/src/sgml/release-9.6.sgml | 447 +++++++++++++++++++--------------- doc/src/sgml/spgist.sgml | 1 + 3 files changed, 264 insertions(+), 212 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 5c1c4f69fb..5148095fb3 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -18580,7 +18580,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_drop(node_name text) - void + void Delete a previously created replication origin, including any @@ -18612,7 +18612,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_session_setup(node_name text) - void + void Mark the current session as replaying from the given @@ -18630,7 +18630,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_session_reset() - void + void Cancel the effects @@ -18679,7 +18679,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_xact_setup(origin_lsn pg_lsn, origin_timestamp timestamptz) - void + void Mark the current transaction as replaying a transaction that has @@ -18698,7 +18698,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_xact_reset() - void + void Cancel the effects of @@ -18714,7 +18714,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_advance(node_name text, pos pg_lsn) - void + void Set replication progress for the given node to the given @@ -19174,7 +19174,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); - brin_summarize_new_values(index_oid regclass) + brin_summarize_new_values(index regclass) integer summarize page ranges not already summarized @@ -19191,8 +19191,8 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); - brin_summarize_new_values receives a BRIN index OID as - argument and inspects the index to find page ranges in the base table + brin_summarize_new_values accepts the OID or name of a + BRIN index and inspects the index to find page ranges in the base table that are not currently summarized by the index; for any such range it creates a new summary index tuple by scanning the table pages. It returns the number of new page range summaries that were inserted @@ -19201,12 +19201,12 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); gin_clean_pending_list accepts the OID or name of - a GIN index and cleans up the pending list of the specified GIN index + a GIN index and cleans up the pending list of the specified index by moving entries in it to the main GIN data structure in bulk. - It returns the number of pages cleaned up from the pending list. - Note that if the argument is a GIN index built with fastupdate - option disabled, the cleanup does not happen and the return value is 0 - because the index doesn't have a pending list. + It returns the number of pages removed from the pending list. + Note that if the argument is a GIN index built with + the fastupdate option disabled, no cleanup happens and the + return value is 0, because the index doesn't have a pending list. Please see and for details of the pending list and fastupdate option. diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 6ec7583485..895d88e768 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -23,33 +23,33 @@ - Parallel sequential scans, joins and aggregates + Parallel execution of sequential scans, joins and aggregates - Eliminate repetitive scanning of old data by autovacuum + Autovacuum no longer performs repetitive scanning of old data - Synchronous replication now allows multiple standby servers, for + Synchronous replication now allows multiple standby servers for increased reliability - Allow full-text search for phrases (multiple adjacent words) + Full-text search can now search for phrases (multiple adjacent words) - Support foreign/remote joins, sorts, and UPDATEs in - postgres_fdw + postgres_fdw now supports remote joins, sorts, + UPDATEs, and DELETEs @@ -87,25 +87,6 @@ - - Change the column name in the - information_schema.routines - view from result_cast_character_set_name - to result_cast_char_set_name (Clément - Prévost) - - - - The SQL:2011 standard specifies the longer name, but that appears - to be a mistake, because adjacent column names use the shorter - style, as do other information_schema views. - - - - - @@ -117,7 +98,7 @@ Historically a process has only been shown as waiting if it was - waiting for a heavy weight lock. Now waits for light weight locks + waiting for a heavyweight lock. Now waits for lightweight locks and buffer pins are also shown in pg_stat_activity. Also, the type of lock being waited for is now visible. These changes replace the waiting column with @@ -149,18 +130,18 @@ Make extract() behave - more reasonably with infinite inputs (Vitaly Burovoy) + more reasonably with infinite inputs (Vitaly Burovoy) Historically the extract() function just returned zero given an infinite timestamp, regardless of the given - unit name. Make it return infinity + field name. Make it return infinity or -infinity as appropriate when the requested field is one that is monotonically increasing (e.g, year, epoch), or NULL when it is not (e.g., day, hour). Also, - throw the expected error for bad unit names. + throw the expected error for bad field names. @@ -186,8 +167,8 @@ This commit is also listed under libpq and psql 2016-03-29 [61d66c44f] Fix support of digits in email/hostnames. --> - Fix text search parser to allow leading digits in email - and host tokens (Artur Zakirov) + Fix the default text search parser to allow leading digits + in email and host tokens (Artur Zakirov) @@ -205,18 +186,18 @@ This commit is also listed under libpq and psql 2016-03-16 [9a206d063] Improve script generating unaccent rules --> - Extend contrib/unaccent's standard - unaccent.rules file to handle all diacritics - known to Unicode, and expand ligatures correctly (Thomas Munro, + Extend contrib/unaccent's + standard unaccent.rules file to handle all diacritics + known to Unicode, and to expand ligatures correctly (Thomas Munro, Léonard Benedetti) - The previous version omitted some less-common letters with - diacritic marks. It now also expands ligatures into separate - letters. Installations that use this rules file may wish to - rebuild tsvector columns and indexes that depend on - the result. + The previous version neglected to convert some less-common letters + with diacritic marks. Also, ligatures are now expanded into + separate letters. Installations that use this rules file may wish + to rebuild tsvector columns and indexes that depend on the + result. @@ -258,17 +239,38 @@ This commit is also listed under libpq and psql + + Change a column name in the + information_schema.routines + view from result_cast_character_set_name + to result_cast_char_set_name (Clément + Prévost) + + + + The SQL:2011 standard specifies the longer name, but that appears + to be a mistake, because adjacent column names use the shorter + style, as do other information_schema views. + + + + + - Support multiple and - command-line options (Pavel Stehule, Catalin Iacob) + psql's option no longer implies + + (Pavel Stehule, Catalin Iacob) - To allow this with sane behavior, one backwards incompatibility - had to be introduced: no longer implies - . + Write (or its + abbreviation ) explicitly to obtain the old + behavior. Scripts modified this way will still work with old + versions of psql. @@ -277,7 +279,7 @@ This commit is also listed under libpq and psql 2015-07-02 [5671aaca8] Improve pg_restore's -t switch to match all types of rel --> - Improve pg_restore's switch to + Improve pg_restore's option to match all types of relations, not only plain tables (Craig Ringer) @@ -287,7 +289,7 @@ This commit is also listed under libpq and psql 2016-02-12 [59a884e98] Change delimiter used for display of NextXID --> - Change the display format of NextXID in + Change the display format used for NextXID in pg_controldata and related places (Joe Conway, Bruce Momjian) @@ -383,8 +385,8 @@ This commit is also listed under libpq and psql 2015-09-16 [7aea8e4f2] Determine whether it's safe to attempt a parallel plan f --> - Provide infrastructure for marking the parallel-safe status of - functions (Robert Haas, Amit Kapila) + Provide infrastructure for marking the parallel-safety status of + functions (Robert Haas, Amit Kapila) @@ -432,8 +434,12 @@ This commit is also listed under libpq and psql Add gin_clean_pending_list() function to allow manual invocation of pending-list cleanup for a - GIN index, separately from vacuuming or analyzing the parent table - (Jeff Janes) + GIN index (Jeff Janes) + + + + Formerly, such cleanup happened only as a byproduct of vacuuming or + analyzing the parent table. @@ -711,7 +717,7 @@ This commit is also listed under libpq and psql If necessary, vacuum can be forced to process all-frozen pages using the new DISABLE_PAGE_SKIPPING option. - Normally, this should never be needed but it might help in + Normally this should never be needed, but it might help in recovering from visibility-map corruption. @@ -774,7 +780,7 @@ This commit is also listed under libpq and psql 2016-02-11 [d4c3a156c] Remove GROUP BY columns that are functionally dependent --> - Avoid computing GROUP BY columns if they are + Ignore GROUP BY columns that are functionally dependent on other columns (David Rowley) @@ -788,43 +794,21 @@ This commit is also listed under libpq and psql - - When appropriate, postpone evaluation of SELECT - output expressions until after an ORDER BY sort - (Konstantin Knizhnik) - - - - This change ensures that volatile or expensive functions in the - output list are executed in the order suggested by ORDER - BY, and that they are not evaluated more times than required - when there is a LIMIT clause. Previously, these - properties held if the ordering was performed by an index scan or - pre-merge-join sort, but not if it was performed by a top-level - sort. - - - - - Allow use of an index-only scan on a partial index when the index's WHERE - clause references columns which are not indexed (Tomas Vondra, + clause references columns that are not indexed (Tomas Vondra, Kyotaro Horiguchi) - For example, CREATE INDEX tidx_partial ON t(b) WHERE a - > 0 could not previously be used for an index-only scan by a - query that only referenced a in its WHERE - clause because a is not an indexed value like - b is. + For example, an index defined by CREATE INDEX tidx_partial + ON t(b) WHERE a > 0 can now be used for an index-only scan by + a query that specifies WHERE a > 0 and does not + otherwise use a. Previously this was disallowed + because a is not listed as an index column. @@ -875,11 +859,12 @@ This commit is also listed under libpq and psql On Linux, sync_file_range() is used for this purpose, - and the feature is on by default on Linux because that function has few - downsides. This sync capability is also available on other platforms - that have msync() or posix_fadvise(), - but those interfaces have some undesirable side-effects so the - feature is disabled by default on non-Linux platforms. + and the feature is on by default on Linux because that function has + few downsides. This flushing capability is also available on other + platforms if they have msync() + or posix_fadvise(), but those interfaces have some + undesirable side-effects so the feature is disabled by default on + non-Linux platforms. @@ -902,8 +887,8 @@ This commit is also listed under libpq and psql - For example, SELECT AVG(x), SUM(x) FROM x can use a - single per-row compuation for both aggregates. + For example, SELECT AVG(x), VARIANCE(x) FROM tab can use + a single per-row computation for both aggregates. @@ -913,9 +898,9 @@ This commit is also listed under libpq and psql --> Speed up visibility tests for recently-created tuples by checking - our transaction snapshot, not pg_clog, to decide - if the source transaction should be considered committed (Jeff - Janes, Tom Lane) + the current transaction's snapshot, not pg_clog, to + decide if the source transaction should be considered committed + (Jeff Janes, Tom Lane) @@ -940,10 +925,11 @@ This commit is also listed under libpq and psql Two-phase commit information is now written only to WAL - during PREPARE TRANSACTION, and read from - WAL during COMMIT PREPARED. A separate - state file is created only if the pending transaction does not - get committed or aborted by the time of the next checkpoint. + during PREPARE TRANSACTION, and will be read back from + WAL during COMMIT PREPARED if that happens + soon thereafter. A separate state file is created only if the + pending transaction does not get committed or aborted by the time + of the next checkpoint. @@ -1142,9 +1128,9 @@ This commit is also listed under libpq and psql This function returns an array of the process IDs of any sessions that are blocking the session with the given process ID. Historically users have obtained such information using a self-join - on the pg_locks view. However, it is unreasonably + on the pg_locks view. However, it is unreasonably tedious to do it that way with any modicum of correctness, and - the addition of parallel queries has made the approach entirely + the addition of parallel queries has made the old approach entirely impractical, since locks might be held or awaited by child worker processes rather than the session's main process. @@ -1181,7 +1167,7 @@ This commit is also listed under libpq and psql - The memory usage dump output to the postmaster log during an + The memory usage dump that is output to the postmaster log during an out-of-memory failure now summarizes statistics when there are a large number of memory contexts, rather than possibly generating a very large report. There is also a grand total @@ -1203,7 +1189,8 @@ This commit is also listed under libpq and psql 2016-04-08 [34c33a1f0] Add BSD authentication method. --> - Add an bsd authentication method to allow the use of + Add a bsd authentication + method to allow use of the BSD Authentication service for PostgreSQL client authentication (Marisa Emerson) @@ -1219,9 +1206,10 @@ This commit is also listed under libpq and psql 2016-04-08 [2f1d2b7a7] Set PAM_RHOST item for PAM authentication --> - When using PAM authentication, provide the client - IP address or host name to PAM modules via the - PAM_RHOST item (Grzegorz Sampolski) + When using PAM + authentication, provide the client IP address or host name + to PAM modules via the PAM_RHOST item + (Grzegorz Sampolski) @@ -1230,8 +1218,8 @@ This commit is also listed under libpq and psql 2016-01-07 [5e0b5dcab] Provide more detail in postmaster log for password authe --> - Provide detail in the postmaster log during more password - authentication failures (Tom Lane) + Provide detail in the postmaster log for more types of password + authentication failure (Tom Lane) @@ -1245,8 +1233,8 @@ This commit is also listed under libpq and psql 2015-09-06 [643beffe8] Support RADIUS passwords up to 128 characters --> - Support RADIUS passwords up to 128 characters long - (Marko Tiikkaja) + Support RADIUS passwords + up to 128 characters long (Marko Tiikkaja) @@ -1255,7 +1243,8 @@ This commit is also listed under libpq and psql 2016-04-08 [35e2e357c] Add authentication parameters compat_realm and upn_usena --> - Add new SSPI authentication parameters + Add new SSPI + authentication parameters compat_realm and upn_username to control whether NetBIOS or Kerberos realm names and user names are used during SSPI @@ -1302,30 +1291,13 @@ This commit is also listed under libpq and psql - - Add configure option - - - This allows the use of systemd service units of - type notify, which greatly simplifies the management - of PostgreSQL under systemd. - - - - - - Add log_line_prefix option %n to print - the current time as a Unix epoch, with milliseconds (Tomas Vondra, - Jeff Davis) + Add option %n to + print the current time in Unix epoch form, with milliseconds (Tomas + Vondra, Jeff Davis) @@ -1362,6 +1334,23 @@ This commit is also listed under libpq and psql + + Add configure option + + + This allows the use of systemd service units of + type notify, which greatly simplifies the management + of PostgreSQL under systemd. + + + + + @@ -1407,8 +1396,8 @@ This commit is also listed under libpq and psql but that is unsafe and inefficient. It also prevents a new postmaster from being started until the last old backend has exited. Backends will detect postmaster death when waiting for - client I/O, so the exit will not be instantaneous, but it in most - cases should happen no later than the end of the current query. + client I/O, so the exit will not be instantaneous, but it should + happen no later than the end of the current query. @@ -1541,7 +1530,8 @@ XXX this is pending backpatch, may need to remove --> Add a option to - pg_basebackup (Peter Eisentraut) + pg_basebackup + (Peter Eisentraut) @@ -1612,6 +1602,28 @@ XXX this is pending backpatch, may need to remove + + When appropriate, postpone evaluation of SELECT + output expressions until after an ORDER BY sort + (Konstantin Knizhnik) + + + + This change ensures that volatile or expensive functions in the + output list are executed in the order suggested by ORDER + BY, and that they are not evaluated more times than required + when there is a LIMIT clause. Previously, these + properties held if the ordering was performed by an index scan or + pre-merge-join sort, but not if it was performed by a top-level + sort. + + + + + @@ -1663,8 +1675,8 @@ XXX this is pending backpatch, may need to remove Previously, the foreign join pushdown infrastructure left the question of security entirely up to individual foreign data - wrappers, but it would be too easy for an FDW to - inadvertently open up subtle security hole. So, make it the core + wrappers, but that made it too easy for an FDW to + inadvertently create subtle security holes. So, make it the core code's job to determine which role ID will access each table, and do not attempt join pushdown unless the role is the same for all relevant relations. @@ -1707,7 +1719,7 @@ XXX this is pending backpatch, may need to remove This command allows a database object to be marked as depending - on an extension, so that it will be automatically dropped if + on an extension, so that it will be dropped automatically if the extension is dropped (without needing CASCADE). However, the object is not part of the extension, and thus will be dumped separately by pg_dump. @@ -1777,8 +1789,8 @@ XXX this is pending backpatch, may need to remove --> Add a CASCADE option to CREATE - EXTENSION to automatically create extensions it depends - on (Petr Jelínek) + EXTENSION to automatically create any extensions the + requested one depends on (Petr Jelínek) @@ -1803,7 +1815,7 @@ XXX this is pending backpatch, may need to remove - This is possible because the table has no existing rows. This matches + This is safe because the table has no existing rows. This matches the longstanding behavior of FOREIGN KEY constraints. @@ -1918,10 +1930,10 @@ XXX this is pending backpatch, may need to remove 2016-06-27 [6734a1cac] Change predecence of phrase operator. --> - Improve full-text search to support searching for phrases, that - is, lexemes appearing adjacent to each other in a specific order, - or with a specified distance between them (Teodor Sigaev, Oleg - Bartunov, Dmitry Ivanov) + Improve full-text search to support + searching for phrases, that is, lexemes appearing adjacent to each + other in a specific order, or with a specified distance between + them (Teodor Sigaev, Oleg Bartunov, Dmitry Ivanov) @@ -2001,9 +2013,10 @@ XXX this is pending backpatch, may need to remove 2016-03-17 [f4ceed6ce] Improve support of Hunspell --> - Upgrade the ispell dictionary to handle modern - Hunspell files and support more languages - (Artur Zakirov) + Upgrade + the ispell + dictionary type to handle modern Hunspell files and + support more languages (Artur Zakirov) @@ -2012,7 +2025,9 @@ XXX this is pending backpatch, may need to remove 2015-10-30 [12c9a0400] Implement lookbehind constraints in our regular-expressi --> - Implement look-behind constraints in regular expressions (Tom Lane) + Implement look-behind constraints + in regular expressions + (Tom Lane) @@ -2044,7 +2059,7 @@ XXX this is pending backpatch, may need to remove 2015-11-07 [c5e86ea93] Add "xid <> xid" and "xid <> int4" operators. --> - Add transaction id operators xid <> + Add transaction ID operators xid <> xid and xid <> int4, for consistency with the corresponding equality operators (Michael Paquier) @@ -2090,8 +2105,10 @@ XXX this is pending backpatch, may need to remove 2016-01-05 [abb173392] Add scale(numeric) --> - Add a scale(numeric) function to extract the display - scale of a numeric value (Marko Tiikkaja) + Add a scale(numeric) + function to extract the display scale of a numeric value + (Marko Tiikkaja) @@ -2109,7 +2126,7 @@ XXX this is pending backpatch, may need to remove measures its argument in degrees, whereas sin() measures in radians. These functions go to some lengths to deliver exact results for values where an exact result can be - expected, e.g. sind(30) = 0.5. + expected, for instance sind(30) = 0.5. @@ -2125,9 +2142,9 @@ XXX this is pending backpatch, may need to remove The POSIX standard says that these functions should - return NaN for NaN input, and should throw an error for - out-of-range inputs including infinity. Previously our - behavior varied across platforms. + return NaN for NaN input, and should throw + an error for out-of-range inputs including infinity. + Previously our behavior varied across platforms. @@ -2136,9 +2153,10 @@ XXX this is pending backpatch, may need to remove 2016-03-29 [e511d878f] Allow to_timestamp(float8) to convert float infinity to --> - Make to_timestamp(float8) convert float - infinity to timestamp infinity (Vitaly - Burovoy) + Make to_timestamp(float8) + convert float infinity to + timestamp infinity (Vitaly Burovoy) @@ -2170,11 +2188,12 @@ XXX this is pending backpatch, may need to remove 2015-09-17 [9acb9007d] Fix oversight in tsearch type check --> - Allow ts_stat_sql() and - tsvector_update_trigger() to operate on values that - are of types binary-compatible with the expected argument type, - not just that argument type; for example allow citext - where text is expected (Teodor Sigaev) + Allow ts_stat() + and tsvector_update_trigger() + to operate on values that are of types binary-compatible with the + expected argument type, not only that argument type; for example + allow citext where text is expected (Teodor + Sigaev) @@ -2216,8 +2235,9 @@ XXX this is pending backpatch, may need to remove In to_number(), - interpret V as dividing by 10 to the power of the - number of digits following V (Bruce Momjian) + interpret a V format code as dividing by 10 to the + power of the number of digits following V (Bruce + Momjian) @@ -2231,8 +2251,10 @@ XXX this is pending backpatch, may need to remove 2016-01-05 [ea0d494da] Make the to_reg*() functions accept text not cstring. --> - Make the to_reg*() functions accept type text - not cstring (Petr Korobeinikov) + Make the to_reg*() + functions accept type text not cstring + (Petr Korobeinikov) @@ -2289,7 +2311,7 @@ XXX this is pending backpatch, may need to remove This allows avoiding an error for an unrecognized parameter - name, and instead return a NULL. + name, instead returning a NULL. @@ -2348,7 +2370,7 @@ XXX this is pending backpatch, may need to remove In PL/pgSQL, detect mismatched CONTINUE and EXIT statements while - compiling PL/pgSQL functions, rather than at execution time + compiling a function, rather than at execution time (Jim Nasby) @@ -2453,8 +2475,9 @@ XXX this is pending backpatch, may need to remove 2016-08-26 [e796d0aba] Add a nonlocalized version of the severity field to clie --> - Add a nonlocalized version of the severity field in error and notice - messages (Tom Lane) + Add a nonlocalized version of + the severity field in + error and notice messages (Tom Lane) @@ -2495,6 +2518,8 @@ This commit is also listed under psql and PL/pgSQL + This is done with the new function PQresultVerboseErrorMessage(). This supports psql's new \errverbose feature, and may be useful for other clients as well. @@ -2541,8 +2566,10 @@ This commit is also listed under psql and PL/pgSQL 2015-09-14 [d02426029] Check existency of table/schema for -t/-n option (pg_dum --> - Add a @@ -2594,6 +2621,22 @@ This commit is also listed under psql and PL/pgSQL + + Support multiple and + command-line options (Pavel Stehule, Catalin Iacob) + + + + The specified operations are carried out in the order in which the + options are given, and then psql terminates. + + + + + @@ -2679,7 +2722,7 @@ This commit is also listed under psql and PL/pgSQL 2016-06-15 [9901d8ac2] Use strftime("%c") to format timestamps in psql's \watch --> - Improve the headers output of the \watch command + Improve the headers output by the \watch command (Michael Paquier, Tom Lane) @@ -2790,8 +2833,9 @@ This commit is also listed under libpq and PL/pgSQL This change allows SQL commands in scripts to span multiple lines. Existing custom scripts will need to be modified to add a semicolon - at the end of each line if missing. (Doing so does not break - the script for use with older versions of pgbench.) + at the end of each line that does not have one already. (Doing so + does not break the script for use with older versions + of pgbench.) @@ -2960,8 +3004,9 @@ This commit is also listed under libpq and PL/pgSQL 2015-12-01 [e50cda784] Use pg_rewind when target timeline was switched --> - Improve pg_rewind so that it can work when the - target timeline changes (Alexander Korotkov) + Improve pg_rewind + so that it can work when the target timeline changes (Alexander + Korotkov) @@ -3095,8 +3140,9 @@ This commit is also listed under libpq and PL/pgSQL 2016-08-13 [ed0097e4f] Add SQL-accessible functions for inspecting index AM pro --> - Restructure index access method API to hide most of - it at the C level (Alexander Korotkov, Andrew Gierth) + Restructure index access + method API to hide most of it at + the C level (Alexander Korotkov, Andrew Gierth) @@ -3118,9 +3164,11 @@ This commit is also listed under libpq and PL/pgSQL 2016-04-06 [6c268df12] Add new catalog called pg_init_privs --> - Add pg_init_privs system catalog to hold original - privileges of initdb-created and extension-created - objects (Stephen Frost) + Add pg_init_privs + system catalog to hold original privileges + of initdb-created and extension-created objects + (Stephen Frost) @@ -3315,7 +3363,7 @@ This commit is also listed under libpq and PL/pgSQL This is somewhat like the reconstructed value, but it - could be any arbitrary chunk of data, it need not be of the same + could be any arbitrary chunk of data, not necessarily of the same data type as the indexed column. @@ -3367,9 +3415,10 @@ This commit is also listed under libpq and PL/pgSQL 2016-03-13 [7a8d87483] Rename auto_explain.sample_ratio to sample_rate --> - Add configuration parameter auto_explain.sample_rate - to allow contrib/auto_explain to capture just a - configurable fraction of all queries (Craig Ringer, Julien Rouhaud) + Add configuration parameter auto_explain.sample_rate to + allow contrib/auto_explain + to capture just a configurable fraction of all queries (Craig + Ringer, Julien Rouhaud) @@ -3383,9 +3432,9 @@ This commit is also listed under libpq and PL/pgSQL 2016-04-01 [9ee014fc8] Bloom index contrib module --> - Add contrib/bloom module that implements an index - access method based on Bloom filtering (Teodor Sigaev, Alexander - Korotkov) + Add contrib/bloom module that + implements an index access method based on Bloom filtering (Teodor + Sigaev, Alexander Korotkov) @@ -3401,9 +3450,9 @@ This commit is also listed under libpq and PL/pgSQL 2015-12-28 [81ee726d8] Code and docs review for cube kNN support. --> - In contrib/cube, introduce distance operators for - cubes, and support kNN-style searches in GiST indexes on cube - columns (Stas Kelvich) + In contrib/cube, introduce + distance operators for cubes, and support kNN-style searches in + GiST indexes on cube columns (Stas Kelvich) @@ -3434,8 +3483,9 @@ This commit is also listed under libpq and PL/pgSQL --> Add selectivity estimation functions for - contrib/intarray operators to improve plans for - queries using those operators (Yury Zhuravlev, Alexander Korotkov) + contrib/intarray operators + to improve plans for queries using those operators (Yury Zhuravlev, + Alexander Korotkov) @@ -3470,7 +3520,8 @@ This commit is also listed under libpq and PL/pgSQL --> Add support for word similarity to - contrib/pg_trgm (Alexander Korotkov, Artur Zakirov) + contrib/pg_trgm + (Alexander Korotkov, Artur Zakirov) @@ -3486,9 +3537,8 @@ This commit is also listed under libpq and PL/pgSQL --> Add configuration parameter - pg_trgm.similarity_threshold for contrib/pg_trgm's similarity - threshold (Artur Zakirov) + pg_trgm.similarity_threshold for + contrib/pg_trgm's similarity threshold (Artur Zakirov) @@ -3525,8 +3575,9 @@ This commit is also listed under libpq and PL/pgSQL 2016-06-17 [71d05a2c7] pg_visibility: Add pg_truncate_visibility_map function. --> - Add contrib/pg_visibility module to allow examining - table visibility maps (Robert Haas) + Add contrib/pg_visibility module + to allow examining table visibility maps (Robert Haas) @@ -3545,7 +3596,7 @@ This commit is also listed under libpq and PL/pgSQL - <filename>postgres_fdw</> + <link linkend="postgres-fdw"><filename>postgres_fdw</></> @@ -3597,7 +3648,7 @@ This commit is also listed under libpq and PL/pgSQL - Formerly, this involved sending a SELECT FOR UPDATE + Formerly, remote updates involved sending a SELECT FOR UPDATE command and then updating or deleting the selected rows one-by-one. While that is still necessary if the operation requires any local processing, it can now be done remotely if all elements of the diff --git a/doc/src/sgml/spgist.sgml b/doc/src/sgml/spgist.sgml index d60aa23f33..cd4a8d07c4 100644 --- a/doc/src/sgml/spgist.sgml +++ b/doc/src/sgml/spgist.sgml @@ -674,6 +674,7 @@ typedef struct spgInnerConsistentOut However, any output traverse values pointed to by the traversalValues array should be allocated in traversalMemoryContext. + Each traverse value must be a single palloc'd chunk. From b899ccbb49cbcf8431b3af43fcf3faf91e6a28c6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 28 Aug 2016 17:44:29 -0400 Subject: [PATCH 079/871] Fix stray reference to the old genbki.sh script. Per Tomas Vondra. --- src/include/catalog/pg_foreign_table.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/catalog/pg_foreign_table.h b/src/include/catalog/pg_foreign_table.h index e7b925b7c9..34690f3808 100644 --- a/src/include/catalog/pg_foreign_table.h +++ b/src/include/catalog/pg_foreign_table.h @@ -9,7 +9,7 @@ * src/include/catalog/pg_foreign_table.h * * NOTES - * the genbki.sh script reads this file and generates .bki + * the genbki.pl script reads this file and generates .bki * information from the DATA() statements. * *------------------------------------------------------------------------- From bab7823a49bb210b8920ae59e5126d27e4d63833 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Mon, 29 Aug 2016 14:34:58 +0900 Subject: [PATCH 080/871] Fix pg_xlogdump so that it handles cross-page XLP_FIRST_IS_CONTRECORD record. Previously pg_xlogdump failed to dump the contents of the WAL file if the file starts with the continuation WAL record which spans more than one pages. Since pg_xlogdump assumed that the continuation record always fits on a page, it could not find the valid WAL record to start reading from in that case. This patch changes pg_xlogdump so that it can handle a continuation WAL record which crosses a page boundary and find the valid record to start reading from. Back-patch to 9.3 where pg_xlogdump was introduced. Author: Pavan Deolasee Reviewed-By: Michael Paquier and Craig Ringer Discussion: CABOikdPsPByMiG6J01DKq6om2+BNkxHTPkOyqHM2a4oYwGKsqQ@mail.gmail.com --- src/backend/access/transam/xlogreader.c | 91 +++++++++++++++++-------- 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index dcf747c633..f2da505892 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -866,46 +866,83 @@ XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) { XLogReaderState saved_state = *state; - XLogRecPtr targetPagePtr; XLogRecPtr tmpRecPtr; - int targetRecOff; XLogRecPtr found = InvalidXLogRecPtr; - uint32 pageHeaderSize; XLogPageHeader header; - int readLen; char *errormsg; Assert(!XLogRecPtrIsInvalid(RecPtr)); - targetRecOff = RecPtr % XLOG_BLCKSZ; + /* + * skip over potential continuation data, keeping in mind that it may span + * multiple pages + */ + tmpRecPtr = RecPtr; + while (true) + { + XLogRecPtr targetPagePtr; + int targetRecOff; + uint32 pageHeaderSize; + int readLen; - /* scroll back to page boundary */ - targetPagePtr = RecPtr - targetRecOff; + /* + * Compute targetRecOff. It should typically be equal or greater than + * short page-header since a valid record can't start anywhere before + * that, except when caller has explicitly specified the offset that + * falls somewhere there or when we are skipping multi-page + * continuation record. It doesn't matter though because + * ReadPageInternal() is prepared to handle that and will read at least + * short page-header worth of data + */ + targetRecOff = tmpRecPtr % XLOG_BLCKSZ; - /* Read the page containing the record */ - readLen = ReadPageInternal(state, targetPagePtr, targetRecOff); - if (readLen < 0) - goto err; + /* scroll back to page boundary */ + targetPagePtr = tmpRecPtr - targetRecOff; - header = (XLogPageHeader) state->readBuf; + /* Read the page containing the record */ + readLen = ReadPageInternal(state, targetPagePtr, targetRecOff); + if (readLen < 0) + goto err; - pageHeaderSize = XLogPageHeaderSize(header); + header = (XLogPageHeader) state->readBuf; - /* make sure we have enough data for the page header */ - readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize); - if (readLen < 0) - goto err; + pageHeaderSize = XLogPageHeaderSize(header); - /* skip over potential continuation data */ - if (header->xlp_info & XLP_FIRST_IS_CONTRECORD) - { - /* record headers are MAXALIGN'ed */ - tmpRecPtr = targetPagePtr + pageHeaderSize - + MAXALIGN(header->xlp_rem_len); - } - else - { - tmpRecPtr = targetPagePtr + pageHeaderSize; + /* make sure we have enough data for the page header */ + readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize); + if (readLen < 0) + goto err; + + /* skip over potential continuation data */ + if (header->xlp_info & XLP_FIRST_IS_CONTRECORD) + { + /* + * If the length of the remaining continuation data is more than + * what can fit in this page, the continuation record crosses over + * this page. Read the next page and try again. xlp_rem_len in the + * next page header will contain the remaining length of the + * continuation data + * + * Note that record headers are MAXALIGN'ed + */ + if (MAXALIGN(header->xlp_rem_len) > (XLOG_BLCKSZ - pageHeaderSize)) + tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + else + { + /* + * The previous continuation record ends in this page. Set + * tmpRecPtr to point to the first valid record + */ + tmpRecPtr = targetPagePtr + pageHeaderSize + + MAXALIGN(header->xlp_rem_len); + break; + } + } + else + { + tmpRecPtr = targetPagePtr + pageHeaderSize; + break; + } } /* From bd082231edbaf25626a023913394b611fe7928e8 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Mon, 29 Aug 2016 16:06:40 +0900 Subject: [PATCH 081/871] Fix typos in comments. --- src/backend/access/brin/brin_inclusion.c | 2 +- src/timezone/localtime.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/access/brin/brin_inclusion.c b/src/backend/access/brin/brin_inclusion.c index 0ae7a72996..c647be0ced 100644 --- a/src/backend/access/brin/brin_inclusion.c +++ b/src/backend/access/brin/brin_inclusion.c @@ -431,7 +431,7 @@ brin_inclusion_consistent(PG_FUNCTION_ARGS) * It is straightforward to support the equality strategies with * the contains operator. Generally, inequality strategies do not * make much sense for the types which will be used with the - * inclusion BRIN family of opclasses, but is is possible to + * inclusion BRIN family of opclasses, but is possible to * implement them with logical negation of the left-of and * right-of operators. * diff --git a/src/timezone/localtime.c b/src/timezone/localtime.c index a14215d6bd..d004e5ebe2 100644 --- a/src/timezone/localtime.c +++ b/src/timezone/localtime.c @@ -489,7 +489,7 @@ tzloadbody(char const * name, char *canonname, struct state * sp, bool doextend, } /* - * If type 0 is is unused in transitions, it's the type to use for early + * If type 0 is unused in transitions, it's the type to use for early * times. */ for (i = 0; i < sp->timecnt; ++i) From 49340627f9821e447f135455d942f7d5e96cae6d Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Mon, 29 Aug 2016 12:16:18 +0100 Subject: [PATCH 082/871] Fix pg_receivexlog --synchronous Make pg_receivexlog work correctly with --synchronous without slots Backpatch to 9.5 Gabriele Bartolini, reviewed by Michael Paquier and Simon Riggs --- src/bin/pg_basebackup/receivelog.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index 595213f042..062730b6b4 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -503,26 +503,28 @@ ReceiveXlogStream(PGconn *conn, StreamCtl *stream) if (!CheckServerVersionForStreaming(conn)) return false; + /* + * Decide whether we want to report the flush position. If we report + * the flush position, the primary will know what WAL we'll + * possibly re-request, and it can then remove older WAL safely. + * We must always do that when we are using slots. + * + * Reporting the flush position makes one eligible as a synchronous + * replica. People shouldn't include generic names in + * synchronous_standby_names, but we've protected them against it so + * far, so let's continue to do so unless specifically requested. + */ if (replication_slot != NULL) { - /* - * Report the flush position, so the primary can know what WAL we'll - * possibly re-request, and remove older WAL safely. - * - * We only report it when a slot has explicitly been used, because - * reporting the flush position makes one eligible as a synchronous - * replica. People shouldn't include generic names in - * synchronous_standby_names, but we've protected them against it so - * far, so let's continue to do so in the situations when possible. If - * they've got a slot, though, we need to report the flush position, - * so that the master can remove WAL. - */ reportFlushPosition = true; sprintf(slotcmd, "SLOT \"%s\" ", replication_slot); } else { - reportFlushPosition = false; + if (stream->synchronous) + reportFlushPosition = true; + else + reportFlushPosition = false; slotcmd[0] = 0; } From cf34fdbbe1452b9e19c0956bc48494889e1b2777 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 29 Aug 2016 09:29:26 -0400 Subject: [PATCH 083/871] Make AllocSetContextCreate throw an error for bad context-size parameters. The previous behavior was to silently change them to something valid. That obscured the bugs fixed in commit ea268cdc9, and generally seems less useful than complaining. Unlike the previous commit, though, we'll do this in HEAD only --- it's a bit too late to be possibly breaking third-party code in 9.6. Discussion: --- src/backend/utils/mmgr/aset.c | 37 ++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c index 43c85234ce..f44e46767b 100644 --- a/src/backend/utils/mmgr/aset.c +++ b/src/backend/utils/mmgr/aset.c @@ -445,6 +445,26 @@ AllocSetContextCreate(MemoryContext parent, { AllocSet set; + /* + * First, validate allocation parameters. (If we're going to throw an + * error, we should do so before the context is created, not after.) We + * somewhat arbitrarily enforce a minimum 1K block size. + */ + if (initBlockSize != MAXALIGN(initBlockSize) || + initBlockSize < 1024) + elog(ERROR, "invalid initBlockSize for memory context: %zu", + initBlockSize); + if (maxBlockSize != MAXALIGN(maxBlockSize) || + maxBlockSize < initBlockSize || + !AllocHugeSizeIsValid(maxBlockSize)) /* must be safe to double */ + elog(ERROR, "invalid maxBlockSize for memory context: %zu", + maxBlockSize); + if (minContextSize != 0 && + (minContextSize != MAXALIGN(minContextSize) || + minContextSize <= ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ)) + elog(ERROR, "invalid minContextSize for memory context: %zu", + minContextSize); + /* Do the type-independent part of context creation */ set = (AllocSet) MemoryContextCreate(T_AllocSetContext, sizeof(AllocSetContext), @@ -452,18 +472,7 @@ AllocSetContextCreate(MemoryContext parent, parent, name); - /* - * Make sure alloc parameters are reasonable, and save them. - * - * We somewhat arbitrarily enforce a minimum 1K block size. - */ - initBlockSize = MAXALIGN(initBlockSize); - if (initBlockSize < 1024) - initBlockSize = 1024; - maxBlockSize = MAXALIGN(maxBlockSize); - if (maxBlockSize < initBlockSize) - maxBlockSize = initBlockSize; - Assert(AllocHugeSizeIsValid(maxBlockSize)); /* must be safe to double */ + /* Save allocation parameters */ set->initBlockSize = initBlockSize; set->maxBlockSize = maxBlockSize; set->nextBlockSize = initBlockSize; @@ -495,9 +504,9 @@ AllocSetContextCreate(MemoryContext parent, /* * Grab always-allocated space, if requested */ - if (minContextSize > ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ) + if (minContextSize > 0) { - Size blksize = MAXALIGN(minContextSize); + Size blksize = minContextSize; AllocBlock block; block = (AllocBlock) malloc(blksize); From 9b7cd59af1afcfbd786921d5cf73befb5fefa2f7 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 29 Aug 2016 20:16:02 +0300 Subject: [PATCH 084/871] Remove support for OpenSSL versions older than 0.9.8. OpenSSL officially only supports 1.0.1 and newer. Some OS distributions still provide patches for 0.9.8, but anything older than that is not interesting anymore. Let's simplify things by removing compatibility code. Andreas Karlsson, with small changes by me. --- contrib/pgcrypto/openssl.c | 152 +---------------------- doc/src/sgml/installation.sgml | 39 ++---- doc/src/sgml/libpq.sgml | 3 +- doc/src/sgml/pgcrypto.sgml | 18 +-- src/backend/libpq/be-secure-openssl.c | 8 +- src/interfaces/libpq/fe-secure-openssl.c | 4 - src/interfaces/libpq/libpq-int.h | 2 +- 7 files changed, 20 insertions(+), 206 deletions(-) diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index 976af70591..ffab5d2bb0 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -46,155 +47,6 @@ #define MAX_KEY (512/8) #define MAX_IV (128/8) -/* - * Compatibility with OpenSSL 0.9.6 - * - * It needs AES and newer DES and digest API. - */ -#if OPENSSL_VERSION_NUMBER >= 0x00907000L - -/* - * Nothing needed for OpenSSL 0.9.7+ - */ - -#include -#else /* old OPENSSL */ - -/* - * Emulate OpenSSL AES. - */ - -#include "rijndael.c" - -#define AES_ENCRYPT 1 -#define AES_DECRYPT 0 -#define AES_KEY rijndael_ctx - -static int -AES_set_encrypt_key(const uint8 *key, int kbits, AES_KEY *ctx) -{ - aes_set_key(ctx, key, kbits, 1); - return 0; -} - -static int -AES_set_decrypt_key(const uint8 *key, int kbits, AES_KEY *ctx) -{ - aes_set_key(ctx, key, kbits, 0); - return 0; -} - -static void -AES_ecb_encrypt(const uint8 *src, uint8 *dst, AES_KEY *ctx, int enc) -{ - memcpy(dst, src, 16); - if (enc) - aes_ecb_encrypt(ctx, dst, 16); - else - aes_ecb_decrypt(ctx, dst, 16); -} - -static void -AES_cbc_encrypt(const uint8 *src, uint8 *dst, int len, AES_KEY *ctx, uint8 *iv, int enc) -{ - memcpy(dst, src, len); - if (enc) - { - aes_cbc_encrypt(ctx, iv, dst, len); - memcpy(iv, dst + len - 16, 16); - } - else - { - aes_cbc_decrypt(ctx, iv, dst, len); - memcpy(iv, src + len - 16, 16); - } -} - -/* - * Emulate DES_* API - */ - -#define DES_key_schedule des_key_schedule -#define DES_cblock des_cblock -#define DES_set_key(k, ks) \ - des_set_key((k), *(ks)) -#define DES_ecb_encrypt(i, o, k, e) \ - des_ecb_encrypt((i), (o), *(k), (e)) -#define DES_ncbc_encrypt(i, o, l, k, iv, e) \ - des_ncbc_encrypt((i), (o), (l), *(k), (iv), (e)) -#define DES_ecb3_encrypt(i, o, k1, k2, k3, e) \ - des_ecb3_encrypt((des_cblock *)(i), (des_cblock *)(o), \ - *(k1), *(k2), *(k3), (e)) -#define DES_ede3_cbc_encrypt(i, o, l, k1, k2, k3, iv, e) \ - des_ede3_cbc_encrypt((i), (o), \ - (l), *(k1), *(k2), *(k3), (iv), (e)) - -/* - * Emulate newer digest API. - */ - -static void -EVP_MD_CTX_init(EVP_MD_CTX *ctx) -{ - memset(ctx, 0, sizeof(*ctx)); -} - -static int -EVP_MD_CTX_cleanup(EVP_MD_CTX *ctx) -{ - px_memset(ctx, 0, sizeof(*ctx)); - return 1; -} - -static int -EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *md, void *engine) -{ - EVP_DigestInit(ctx, md); - return 1; -} - -static int -EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *res, unsigned int *len) -{ - EVP_DigestFinal(ctx, res, len); - return 1; -} -#endif /* old OpenSSL */ - -/* - * Provide SHA2 for older OpenSSL < 0.9.8 - */ -#if OPENSSL_VERSION_NUMBER < 0x00908000L - -#include "sha2.c" -#include "internal-sha2.c" - -typedef void (*init_f) (PX_MD *md); - -static int -compat_find_digest(const char *name, PX_MD **res) -{ - init_f init = NULL; - - if (pg_strcasecmp(name, "sha224") == 0) - init = init_sha224; - else if (pg_strcasecmp(name, "sha256") == 0) - init = init_sha256; - else if (pg_strcasecmp(name, "sha384") == 0) - init = init_sha384; - else if (pg_strcasecmp(name, "sha512") == 0) - init = init_sha512; - else - return PXE_NO_HASH; - - *res = px_alloc(sizeof(PX_MD)); - init(*res); - return 0; -} -#else -#define compat_find_digest(name, res) (PXE_NO_HASH) -#endif - /* * Hashes */ @@ -275,7 +127,7 @@ px_find_digest(const char *name, PX_MD **res) md = EVP_get_digestbyname(name); if (md == NULL) - return compat_find_digest(name, res); + return PXE_NO_HASH; digest = px_alloc(sizeof(*digest)); digest->algo = md; diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index a9968756e6..14a6d57aea 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -252,10 +252,17 @@ su - postgres - You need Kerberos, OpenSSL, - OpenLDAP, and/or - PAM, if you want to support authentication or - encryption using those services. + You need OpenSSL, if you want to support + encrypted client connections. The minimum required version is + 0.9.8. + + + + + + You need Kerberos, OpenLDAP, + and/or PAM, if you want to support authentication + using those services. @@ -2826,30 +2833,6 @@ MANPATH=/usr/lib/scohelp/%L/man:/usr/dt/man:/usr/man:/usr/share/man:scohelp:/usr - - Problems with OpenSSL - - - When you build PostgreSQL with OpenSSL support you might get - compilation errors in the following files: - - src/backend/libpq/crypt.c - src/backend/libpq/password.c - src/interfaces/libpq/fe-auth.c - src/interfaces/libpq/fe-connect.c - - - This is because of a namespace conflict between the standard - /usr/include/crypt.h header and the header - files provided by OpenSSL. - - - - Upgrading your OpenSSL installation to version 0.9.6a fixes this - problem. Solaris 9 and above has a newer version of OpenSSL. - - - configure Complains About a Failed Test Program diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 2f9350b10e..4e34f00e44 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -1238,8 +1238,7 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname If set to 1 (default), data sent over SSL connections will be - compressed (this requires OpenSSL version - 0.9.8 or later). + compressed. If set to 0, compression will be disabled (this requires OpenSSL 1.0.0 or later). This parameter is ignored if a connection without SSL is made, diff --git a/doc/src/sgml/pgcrypto.sgml b/doc/src/sgml/pgcrypto.sgml index c4cefde4f7..bf514aacf3 100644 --- a/doc/src/sgml/pgcrypto.sgml +++ b/doc/src/sgml/pgcrypto.sgml @@ -1184,12 +1184,12 @@ gen_random_uuid() returns uuid SHA224/256/384/512 yes - yes (Note 1) + yes Other digest algorithms no - yes (Note 2) + yes (Note 1) Blowfish @@ -1199,7 +1199,7 @@ gen_random_uuid() returns uuid AES yes - yes (Note 3) + yes DES/3DES/CAST5 @@ -1230,12 +1230,6 @@ gen_random_uuid() returns uuid - - - SHA2 algorithms were added to OpenSSL in version 0.9.8. For - older versions, pgcrypto will use built-in code. - - Any digest algorithm OpenSSL supports is automatically picked up. @@ -1243,12 +1237,6 @@ gen_random_uuid() returns uuid explicitly. - - - AES is included in OpenSSL since version 0.9.7. For - older versions, pgcrypto will use built-in code. - - diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index f6adb155c6..e5f434ca17 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -53,10 +53,8 @@ #include #include -#if SSLEAY_VERSION_NUMBER >= 0x0907000L #include -#endif -#if (OPENSSL_VERSION_NUMBER >= 0x0090800fL) && !defined(OPENSSL_NO_ECDH) +#ifndef OPENSSL_NO_ECDH #include #endif @@ -166,9 +164,7 @@ be_tls_init(void) if (!SSL_context) { -#if SSLEAY_VERSION_NUMBER >= 0x0907000L OPENSSL_config(NULL); -#endif SSL_library_init(); SSL_load_error_strings(); @@ -978,7 +974,7 @@ info_cb(const SSL *ssl, int type, int args) static void initialize_ecdh(void) { -#if (OPENSSL_VERSION_NUMBER >= 0x0090800fL) && !defined(OPENSSL_NO_ECDH) +#ifndef OPENSSL_NO_ECDH EC_KEY *ecdh; int nid; diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index f6ce1c7a13..d8716128ec 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -54,9 +54,7 @@ #endif #include -#if (SSLEAY_VERSION_NUMBER >= 0x00907000L) #include -#endif #ifdef USE_SSL_ENGINE #include #endif @@ -848,9 +846,7 @@ pgtls_init(PGconn *conn) { if (pq_init_ssl_lib) { -#if SSLEAY_VERSION_NUMBER >= 0x00907000L OPENSSL_config(NULL); -#endif SSL_library_init(); SSL_load_error_strings(); } diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 1183323a44..a94ead04ff 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -77,7 +77,7 @@ typedef struct #include #include -#if (SSLEAY_VERSION_NUMBER >= 0x00907000L) && !defined(OPENSSL_NO_ENGINE) +#ifndef OPENSSL_NO_ENGINE #define USE_SSL_ENGINE #endif #endif /* USE_OPENSSL */ From 8e1e3f958fb3749fe01e9f2473f4554859c685a8 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Mon, 29 Aug 2016 18:48:02 -0300 Subject: [PATCH 085/871] =?UTF-8?q?Split=20hash.h=20=E2=86=92=20hash=5Fxlo?= =?UTF-8?q?g.h?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the hash AM is going to be revamped to have WAL, this is a good opportunity to clean up the include file a little bit to avoid including a lot of extra stuff in the future. Author: Amit Kapila --- src/backend/access/hash/hash.c | 1 + src/backend/access/rmgrdesc/hashdesc.c | 2 +- src/backend/access/transam/rmgr.c | 2 +- src/bin/pg_xlogdump/rmgrdesc.c | 2 +- src/include/access/hash.h | 6 ------ src/include/access/hash_xlog.h | 25 +++++++++++++++++++++++++ 6 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 src/include/access/hash_xlog.h diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c index 07496f8156..e3b1eef246 100644 --- a/src/backend/access/hash/hash.c +++ b/src/backend/access/hash/hash.c @@ -19,6 +19,7 @@ #include "postgres.h" #include "access/hash.h" +#include "access/hash_xlog.h" #include "access/relscan.h" #include "catalog/index.h" #include "commands/vacuum.h" diff --git a/src/backend/access/rmgrdesc/hashdesc.c b/src/backend/access/rmgrdesc/hashdesc.c index d37c9b1aae..12e1818fba 100644 --- a/src/backend/access/rmgrdesc/hashdesc.c +++ b/src/backend/access/rmgrdesc/hashdesc.c @@ -14,7 +14,7 @@ */ #include "postgres.h" -#include "access/hash.h" +#include "access/hash_xlog.h" void hash_desc(StringInfo buf, XLogReaderState *record) diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 31c5fd165c..9bb136218d 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -12,7 +12,7 @@ #include "access/gin.h" #include "access/gist_private.h" #include "access/generic_xlog.h" -#include "access/hash.h" +#include "access/hash_xlog.h" #include "access/heapam_xlog.h" #include "access/brin_xlog.h" #include "access/multixact.h" diff --git a/src/bin/pg_xlogdump/rmgrdesc.c b/src/bin/pg_xlogdump/rmgrdesc.c index 017b9c5b34..8fe20ce97e 100644 --- a/src/bin/pg_xlogdump/rmgrdesc.c +++ b/src/bin/pg_xlogdump/rmgrdesc.c @@ -14,7 +14,7 @@ #include "access/generic_xlog.h" #include "access/gin.h" #include "access/gist_private.h" -#include "access/hash.h" +#include "access/hash_xlog.h" #include "access/heapam_xlog.h" #include "access/multixact.h" #include "access/nbtree.h" diff --git a/src/include/access/hash.h b/src/include/access/hash.h index ce314180e6..d9df904555 100644 --- a/src/include/access/hash.h +++ b/src/include/access/hash.h @@ -20,7 +20,6 @@ #include "access/amapi.h" #include "access/itup.h" #include "access/sdir.h" -#include "access/xlogreader.h" #include "fmgr.h" #include "lib/stringinfo.h" #include "storage/bufmgr.h" @@ -365,9 +364,4 @@ extern bool _hash_convert_tuple(Relation index, extern OffsetNumber _hash_binsearch(Page page, uint32 hash_value); extern OffsetNumber _hash_binsearch_last(Page page, uint32 hash_value); -/* hash.c */ -extern void hash_redo(XLogReaderState *record); -extern void hash_desc(StringInfo buf, XLogReaderState *record); -extern const char *hash_identify(uint8 info); - #endif /* HASH_H */ diff --git a/src/include/access/hash_xlog.h b/src/include/access/hash_xlog.h new file mode 100644 index 0000000000..5f941a9dfc --- /dev/null +++ b/src/include/access/hash_xlog.h @@ -0,0 +1,25 @@ +/*------------------------------------------------------------------------- + * + * hash_xlog.h + * header file for Postgres hash AM implementation + * + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/hash_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef HASH_XLOG_H +#define HASH_XLOG_H + +#include "access/hash.h" +#include "access/xlogreader.h" + + +extern void hash_redo(XLogReaderState *record); +extern void hash_desc(StringInfo buf, XLogReaderState *record); +extern const char *hash_identify(uint8 info); + +#endif /* HASH_XLOG_H */ From 37f6fd1eaab698983ca1fb2a036d52381347ac71 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 30 Aug 2016 15:25:01 -0400 Subject: [PATCH 086/871] Fix initdb misbehavior when user mis-enters superuser password. While testing simple_prompt() revisions, I happened to notice that current initdb behaves rather badly when --pwprompt is specified and the user miskeys the second password. It complains about the mismatch, does "rm -rf" on the data directory, and exits. The problem is that since commit c4a8812cf, there's a standalone backend sitting waiting for commands at that point. It gets unhappy about its datadir having gone away, and spews a PANIC message at the user, which is not nice. (And the shell then adds to the mess with meaningless bleating about a core dump...) We don't really want that sort of thing to happen unless there's an internal failure in initdb, which this surely is not. The best fix seems to be to move the collection of the password earlier, so that it's done essentially as part of argument collection, rather than at the rather ad-hoc time it was done before. Back-patch to 9.6 where the problem was introduced. --- src/bin/initdb/initdb.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index aad6ba5639..54d338d013 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -135,6 +135,7 @@ static const char *default_text_search_config = ""; static char *username = ""; static bool pwprompt = false; static char *pwfilename = NULL; +static char *superuser_password = NULL; static const char *authmethodhost = ""; static const char *authmethodlocal = ""; static bool debug = false; @@ -255,7 +256,7 @@ static void test_config_settings(void); static void setup_config(void); static void bootstrap_template1(void); static void setup_auth(FILE *cmdfd); -static void get_set_pwd(FILE *cmdfd); +static void get_su_pwd(void); static void setup_depend(FILE *cmdfd); static void setup_sysviews(FILE *cmdfd); static void setup_description(FILE *cmdfd); @@ -1544,13 +1545,17 @@ setup_auth(FILE *cmdfd) for (line = pg_authid_setup; *line != NULL; line++) PG_CMD_PUTS(*line); + + if (superuser_password) + PG_CMD_PRINTF2("ALTER USER \"%s\" WITH PASSWORD E'%s';\n\n", + username, escape_quotes(superuser_password)); } /* - * get the superuser password if required, and call postgres to set it + * get the superuser password if required */ static void -get_set_pwd(FILE *cmdfd) +get_su_pwd(void) { char *pwd1, *pwd2; @@ -1560,6 +1565,8 @@ get_set_pwd(FILE *cmdfd) /* * Read password from terminal */ + printf("\n"); + fflush(stdout); pwd1 = simple_prompt("Enter new superuser password: ", 100, false); pwd2 = simple_prompt("Enter it again: ", 100, false); if (strcmp(pwd1, pwd2) != 0) @@ -1609,10 +1616,7 @@ get_set_pwd(FILE *cmdfd) } - PG_CMD_PRINTF2("ALTER USER \"%s\" WITH PASSWORD E'%s';\n\n", - username, escape_quotes(pwd1)); - - free(pwd1); + superuser_password = pwd1; } /* @@ -3279,8 +3283,6 @@ initialize_data_directory(void) PG_CMD_OPEN; setup_auth(cmdfd); - if (pwprompt || pwfilename) - get_set_pwd(cmdfd); setup_depend(cmdfd); @@ -3569,6 +3571,9 @@ main(int argc, char *argv[]) else printf(_("Data page checksums are disabled.\n")); + if (pwprompt || pwfilename) + get_su_pwd(); + printf("\n"); initialize_data_directory(); From 9daec77e165de461fca9d5bc3ece86a91aba5804 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 30 Aug 2016 17:02:02 -0400 Subject: [PATCH 087/871] Simplify correct use of simple_prompt(). The previous API for this function had it returning a malloc'd string. That meant that callers had to check for NULL return, which few of them were doing, and it also meant that callers had to remember to free() the string later, which required extra logic in most cases. Instead, make simple_prompt() write into a buffer supplied by the caller. Anywhere that the maximum required input length is reasonably small, which is almost all of the callers, we can just use a local or static array as the buffer instead of dealing with malloc/free. A fair number of callers used "pointer == NULL" as a proxy for "haven't requested the password yet". Maintaining the same behavior requires adding a separate boolean flag for that, which adds back some of the complexity we save by removing free()s. Nonetheless, this nets out at a small reduction in overall code size, and considerably less code than we would have had if we'd added the missing NULL-return checks everywhere they were needed. In passing, clean up the API comment for simple_prompt() and get rid of a very-unnecessary malloc/free in its Windows code path. This is nominally a bug fix, but it does not seem worth back-patching, because the actual risk of an OOM failure in any of these places seems pretty tiny, and all of them are client-side not server-side anyway. This patch is by me, but it owes a great deal to Michael Paquier who identified the problem and drafted a patch for fixing it the other way. Discussion: --- contrib/oid2name/oid2name.c | 13 ++++----- contrib/vacuumlo/vacuumlo.c | 17 ++++++++---- src/bin/initdb/initdb.c | 23 ++++++---------- src/bin/pg_basebackup/nls.mk | 1 + src/bin/pg_basebackup/streamutil.c | 14 +++++----- src/bin/pg_dump/pg_backup_db.c | 35 ++++++++++-------------- src/bin/pg_dump/pg_dumpall.c | 17 ++++++++---- src/bin/pgbench/pgbench.c | 10 ++++--- src/bin/psql/command.c | 25 ++++++++--------- src/bin/psql/startup.c | 16 +++++++---- src/bin/scripts/common.c | 34 +++++++++-------------- src/bin/scripts/createuser.c | 21 ++++++++------ src/bin/scripts/dropuser.c | 7 ++++- src/include/port.h | 3 +- src/port/sprompt.c | 44 ++++++++++++------------------ 15 files changed, 138 insertions(+), 142 deletions(-) diff --git a/contrib/oid2name/oid2name.c b/contrib/oid2name/oid2name.c index e5eeec21c1..5a2aa1dd0e 100644 --- a/contrib/oid2name/oid2name.c +++ b/contrib/oid2name/oid2name.c @@ -261,7 +261,8 @@ PGconn * sql_conn(struct options * my_opts) { PGconn *conn; - char *password = NULL; + bool have_password = false; + char password[100]; bool new_pass; /* @@ -282,7 +283,7 @@ sql_conn(struct options * my_opts) keywords[2] = "user"; values[2] = my_opts->username; keywords[3] = "password"; - values[3] = password; + values[3] = have_password ? password : NULL; keywords[4] = "dbname"; values[4] = my_opts->dbname; keywords[5] = "fallback_application_name"; @@ -302,17 +303,15 @@ sql_conn(struct options * my_opts) if (PQstatus(conn) == CONNECTION_BAD && PQconnectionNeedsPassword(conn) && - password == NULL) + !have_password) { PQfinish(conn); - password = simple_prompt("Password: ", 100, false); + simple_prompt("Password: ", password, sizeof(password), false); + have_password = true; new_pass = true; } } while (new_pass); - if (password) - free(password); - /* check to see that the backend connection was successfully made */ if (PQstatus(conn) == CONNECTION_BAD) { diff --git a/contrib/vacuumlo/vacuumlo.c b/contrib/vacuumlo/vacuumlo.c index 769c805a84..0a9328dc0e 100644 --- a/contrib/vacuumlo/vacuumlo.c +++ b/contrib/vacuumlo/vacuumlo.c @@ -65,13 +65,17 @@ vacuumlo(const char *database, const struct _param * param) long matched; long deleted; int i; - static char *password = NULL; bool new_pass; bool success = true; + static bool have_password = false; + static char password[100]; /* Note: password can be carried over from a previous call */ - if (param->pg_prompt == TRI_YES && password == NULL) - password = simple_prompt("Password: ", 100, false); + if (param->pg_prompt == TRI_YES && !have_password) + { + simple_prompt("Password: ", password, sizeof(password), false); + have_password = true; + } /* * Start the connection. Loop until we have a password if requested by @@ -91,7 +95,7 @@ vacuumlo(const char *database, const struct _param * param) keywords[2] = "user"; values[2] = param->pg_user; keywords[3] = "password"; - values[3] = password; + values[3] = have_password ? password : NULL; keywords[4] = "dbname"; values[4] = database; keywords[5] = "fallback_application_name"; @@ -110,11 +114,12 @@ vacuumlo(const char *database, const struct _param * param) if (PQstatus(conn) == CONNECTION_BAD && PQconnectionNeedsPassword(conn) && - password == NULL && + !have_password && param->pg_prompt != TRI_NO) { PQfinish(conn); - password = simple_prompt("Password: ", 100, false); + simple_prompt("Password: ", password, sizeof(password), false); + have_password = true; new_pass = true; } } while (new_pass); diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 54d338d013..94074928cb 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -1557,8 +1557,8 @@ setup_auth(FILE *cmdfd) static void get_su_pwd(void) { - char *pwd1, - *pwd2; + char pwd1[100]; + char pwd2[100]; if (pwprompt) { @@ -1567,14 +1567,13 @@ get_su_pwd(void) */ printf("\n"); fflush(stdout); - pwd1 = simple_prompt("Enter new superuser password: ", 100, false); - pwd2 = simple_prompt("Enter it again: ", 100, false); + simple_prompt("Enter new superuser password: ", pwd1, sizeof(pwd1), false); + simple_prompt("Enter it again: ", pwd2, sizeof(pwd2), false); if (strcmp(pwd1, pwd2) != 0) { fprintf(stderr, _("Passwords didn't match.\n")); exit_nicely(); } - free(pwd2); } else { @@ -1587,7 +1586,6 @@ get_su_pwd(void) * for now. */ FILE *pwf = fopen(pwfilename, "r"); - char pwdbuf[MAXPGPATH]; int i; if (!pwf) @@ -1596,7 +1594,7 @@ get_su_pwd(void) progname, pwfilename, strerror(errno)); exit_nicely(); } - if (!fgets(pwdbuf, sizeof(pwdbuf), pwf)) + if (!fgets(pwd1, sizeof(pwd1), pwf)) { if (ferror(pwf)) fprintf(stderr, _("%s: could not read password from file \"%s\": %s\n"), @@ -1608,15 +1606,12 @@ get_su_pwd(void) } fclose(pwf); - i = strlen(pwdbuf); - while (i > 0 && (pwdbuf[i - 1] == '\r' || pwdbuf[i - 1] == '\n')) - pwdbuf[--i] = '\0'; - - pwd1 = pg_strdup(pwdbuf); - + i = strlen(pwd1); + while (i > 0 && (pwd1[i - 1] == '\r' || pwd1[i - 1] == '\n')) + pwd1[--i] = '\0'; } - superuser_password = pwd1; + superuser_password = pg_strdup(pwd1); } /* diff --git a/src/bin/pg_basebackup/nls.mk b/src/bin/pg_basebackup/nls.mk index ec466dcaa2..a34ca3d268 100644 --- a/src/bin/pg_basebackup/nls.mk +++ b/src/bin/pg_basebackup/nls.mk @@ -2,3 +2,4 @@ CATALOG_NAME = pg_basebackup AVAIL_LANGUAGES = de es fr it ko pl pt_BR ru zh_CN GETTEXT_FILES = pg_basebackup.c pg_receivexlog.c pg_recvlogical.c receivelog.c streamutil.c ../../common/fe_memutils.c +GETTEXT_TRIGGERS = simple_prompt diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c index 72d8657004..595eaff46a 100644 --- a/src/bin/pg_basebackup/streamutil.c +++ b/src/bin/pg_basebackup/streamutil.c @@ -41,7 +41,8 @@ char *dbport = NULL; char *replication_slot = NULL; char *dbname = NULL; int dbgetpassword = 0; /* 0=auto, -1=never, 1=always */ -static char *dbpassword = NULL; +static bool have_password = false; +static char password[100]; PGconn *conn = NULL; /* @@ -141,24 +142,23 @@ GetConnection(void) } /* If -W was given, force prompt for password, but only the first time */ - need_password = (dbgetpassword == 1 && dbpassword == NULL); + need_password = (dbgetpassword == 1 && !have_password); do { /* Get a new password if appropriate */ if (need_password) { - if (dbpassword) - free(dbpassword); - dbpassword = simple_prompt(_("Password: "), 100, false); + simple_prompt("Password: ", password, sizeof(password), false); + have_password = true; need_password = false; } /* Use (or reuse, on a subsequent connection) password if we have it */ - if (dbpassword) + if (have_password) { keywords[i] = "password"; - values[i] = dbpassword; + values[i] = password; } else { diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c index d2a3de3c5d..3b9cd89b4a 100644 --- a/src/bin/pg_dump/pg_backup_db.c +++ b/src/bin/pg_dump/pg_backup_db.c @@ -134,6 +134,7 @@ _connectDB(ArchiveHandle *AH, const char *reqdb, const char *requser) const char *newdb; const char *newuser; char *password; + char passbuf[100]; bool new_pass; if (!reqdb) @@ -149,13 +150,12 @@ _connectDB(ArchiveHandle *AH, const char *reqdb, const char *requser) ahlog(AH, 1, "connecting to database \"%s\" as user \"%s\"\n", newdb, newuser); - password = AH->savedPassword ? pg_strdup(AH->savedPassword) : NULL; + password = AH->savedPassword; if (AH->promptPassword == TRI_YES && password == NULL) { - password = simple_prompt("Password: ", 100, false); - if (password == NULL) - exit_horribly(modulename, "out of memory\n"); + simple_prompt("Password: ", passbuf, sizeof(passbuf), false); + password = passbuf; } initPQExpBuffer(&connstr); @@ -201,16 +201,14 @@ _connectDB(ArchiveHandle *AH, const char *reqdb, const char *requser) fprintf(stderr, "Connecting to %s as %s\n", newdb, newuser); - if (password) - free(password); - if (AH->promptPassword != TRI_NO) - password = simple_prompt("Password: ", 100, false); + { + simple_prompt("Password: ", passbuf, sizeof(passbuf), false); + password = passbuf; + } else exit_horribly(modulename, "connection needs password\n"); - if (password == NULL) - exit_horribly(modulename, "out of memory\n"); new_pass = true; } } while (new_pass); @@ -225,8 +223,6 @@ _connectDB(ArchiveHandle *AH, const char *reqdb, const char *requser) free(AH->savedPassword); AH->savedPassword = pg_strdup(PQpass(newConn)); } - if (password) - free(password); termPQExpBuffer(&connstr); @@ -258,18 +254,18 @@ ConnectDatabase(Archive *AHX, { ArchiveHandle *AH = (ArchiveHandle *) AHX; char *password; + char passbuf[100]; bool new_pass; if (AH->connection) exit_horribly(modulename, "already connected to a database\n"); - password = AH->savedPassword ? pg_strdup(AH->savedPassword) : NULL; + password = AH->savedPassword; if (prompt_password == TRI_YES && password == NULL) { - password = simple_prompt("Password: ", 100, false); - if (password == NULL) - exit_horribly(modulename, "out of memory\n"); + simple_prompt("Password: ", passbuf, sizeof(passbuf), false); + password = passbuf; } AH->promptPassword = prompt_password; @@ -309,9 +305,8 @@ ConnectDatabase(Archive *AHX, prompt_password != TRI_NO) { PQfinish(AH->connection); - password = simple_prompt("Password: ", 100, false); - if (password == NULL) - exit_horribly(modulename, "out of memory\n"); + simple_prompt("Password: ", passbuf, sizeof(passbuf), false); + password = passbuf; new_pass = true; } } while (new_pass); @@ -332,8 +327,6 @@ ConnectDatabase(Archive *AHX, free(AH->savedPassword); AH->savedPassword = pg_strdup(PQpass(AH->connection)); } - if (password) - free(password); /* check for version mismatch */ _check_database_version(AH); diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 54a9f48200..b5efb46019 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -1884,13 +1884,17 @@ connectDatabase(const char *dbname, const char *connection_string, bool new_pass; const char *remoteversion_str; int my_version; - static char *password = NULL; const char **keywords = NULL; const char **values = NULL; PQconninfoOption *conn_opts = NULL; + static bool have_password = false; + static char password[100]; - if (prompt_password == TRI_YES && !password) - password = simple_prompt("Password: ", 100, false); + if (prompt_password == TRI_YES && !have_password) + { + simple_prompt("Password: ", password, sizeof(password), false); + have_password = true; + } /* * Start the connection. Loop until we have a password if requested by @@ -1970,7 +1974,7 @@ connectDatabase(const char *dbname, const char *connection_string, values[i] = pguser; i++; } - if (password) + if (have_password) { keywords[i] = "password"; values[i] = password; @@ -1998,11 +2002,12 @@ connectDatabase(const char *dbname, const char *connection_string, if (PQstatus(conn) == CONNECTION_BAD && PQconnectionNeedsPassword(conn) && - password == NULL && + !have_password && prompt_password != TRI_NO) { PQfinish(conn); - password = simple_prompt("Password: ", 100, false); + simple_prompt("Password: ", password, sizeof(password), false); + have_password = true; new_pass = true; } } while (new_pass); diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 8027955121..56c37d537e 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -773,8 +773,9 @@ static PGconn * doConnect(void) { PGconn *conn; - static char *password = NULL; bool new_pass; + static bool have_password = false; + static char password[100]; /* * Start the connection. Loop until we have a password if requested by @@ -794,7 +795,7 @@ doConnect(void) keywords[2] = "user"; values[2] = login; keywords[3] = "password"; - values[3] = password; + values[3] = have_password ? password : NULL; keywords[4] = "dbname"; values[4] = dbName; keywords[5] = "fallback_application_name"; @@ -815,10 +816,11 @@ doConnect(void) if (PQstatus(conn) == CONNECTION_BAD && PQconnectionNeedsPassword(conn) && - password == NULL) + !have_password) { PQfinish(conn); - password = simple_prompt("Password: ", 100, false); + simple_prompt("Password: ", password, sizeof(password), false); + have_password = true; new_pass = true; } } while (new_pass); diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 4aaf657cce..38038d415f 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -1089,11 +1089,11 @@ exec_command(const char *cmd, /* \password -- set user password */ else if (strcmp(cmd, "password") == 0) { - char *pw1; - char *pw2; + char pw1[100]; + char pw2[100]; - pw1 = simple_prompt("Enter new password: ", 100, false); - pw2 = simple_prompt("Enter it again: ", 100, false); + simple_prompt("Enter new password: ", pw1, sizeof(pw1), false); + simple_prompt("Enter it again: ", pw2, sizeof(pw2), false); if (strcmp(pw1, pw2) != 0) { @@ -1139,9 +1139,6 @@ exec_command(const char *cmd, if (opt0) free(opt0); } - - free(pw1); - free(pw2); } /* \prompt -- prompt and set variable */ @@ -1173,7 +1170,10 @@ exec_command(const char *cmd, opt = arg1; if (!pset.inputfile) - result = simple_prompt(prompt_text, 4096, true); + { + result = (char *) pg_malloc(4096); + simple_prompt(prompt_text, result, 4096, true); + } else { if (prompt_text) @@ -1747,20 +1747,19 @@ exec_command(const char *cmd, static char * prompt_for_password(const char *username) { - char *result; + char buf[100]; if (username == NULL) - result = simple_prompt("Password: ", 100, false); + simple_prompt("Password: ", buf, sizeof(buf), false); else { char *prompt_text; prompt_text = psprintf(_("Password for user %s: "), username); - result = simple_prompt(prompt_text, 100, false); + simple_prompt(prompt_text, buf, sizeof(buf), false); free(prompt_text); } - - return result; + return pg_strdup(buf); } static bool diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index 111593cd9d..7ce05fbe4a 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -103,7 +103,8 @@ main(int argc, char *argv[]) { struct adhoc_opts options; int successResult; - char *password = NULL; + bool have_password = false; + char password[100]; char *password_prompt = NULL; bool new_pass; @@ -210,7 +211,10 @@ main(int argc, char *argv[]) options.username); if (pset.getPassword == TRI_YES) - password = simple_prompt(password_prompt, 100, false); + { + simple_prompt(password_prompt, password, sizeof(password), false); + have_password = true; + } /* loop until we have a password if requested by backend */ do @@ -226,7 +230,7 @@ main(int argc, char *argv[]) keywords[2] = "user"; values[2] = options.username; keywords[3] = "password"; - values[3] = password; + values[3] = have_password ? password : NULL; keywords[4] = "dbname"; /* see do_connect() */ values[4] = (options.list_dbs && options.dbname == NULL) ? "postgres" : options.dbname; @@ -244,16 +248,16 @@ main(int argc, char *argv[]) if (PQstatus(pset.db) == CONNECTION_BAD && PQconnectionNeedsPassword(pset.db) && - password == NULL && + !have_password && pset.getPassword != TRI_NO) { PQfinish(pset.db); - password = simple_prompt(password_prompt, 100, false); + simple_prompt(password_prompt, password, sizeof(password), false); + have_password = true; new_pass = true; } } while (new_pass); - free(password); free(password_prompt); if (PQstatus(pset.db) == CONNECTION_BAD) diff --git a/src/bin/scripts/common.c b/src/bin/scripts/common.c index 7c1ebe059f..a71cc64a8c 100644 --- a/src/bin/scripts/common.c +++ b/src/bin/scripts/common.c @@ -68,19 +68,19 @@ connectDatabase(const char *dbname, const char *pghost, const char *pgport, const char *progname, bool fail_ok, bool allow_password_reuse) { PGconn *conn; - static char *password = NULL; bool new_pass; + static bool have_password = false; + static char password[100]; if (!allow_password_reuse) + have_password = false; + + if (!have_password && prompt_password == TRI_YES) { - if (password) - free(password); - password = NULL; + simple_prompt("Password: ", password, sizeof(password), false); + have_password = true; } - if (password == NULL && prompt_password == TRI_YES) - password = simple_prompt("Password: ", 100, false); - /* * Start the connection. Loop until we have a password if requested by * backend. @@ -97,7 +97,7 @@ connectDatabase(const char *dbname, const char *pghost, const char *pgport, keywords[2] = "user"; values[2] = pguser; keywords[3] = "password"; - values[3] = password; + values[3] = have_password ? password : NULL; keywords[4] = "dbname"; values[4] = dbname; keywords[5] = "fallback_application_name"; @@ -123,9 +123,8 @@ connectDatabase(const char *dbname, const char *pghost, const char *pgport, prompt_password != TRI_NO) { PQfinish(conn); - if (password) - free(password); - password = simple_prompt("Password: ", 100, false); + simple_prompt("Password: ", password, sizeof(password), false); + have_password = true; new_pass = true; } } while (new_pass); @@ -275,22 +274,15 @@ yesno_prompt(const char *question) for (;;) { - char *resp; + char resp[10]; - resp = simple_prompt(prompt, 1, true); + simple_prompt(prompt, resp, sizeof(resp), true); if (strcmp(resp, _(PG_YESLETTER)) == 0) - { - free(resp); return true; - } - else if (strcmp(resp, _(PG_NOLETTER)) == 0) - { - free(resp); + if (strcmp(resp, _(PG_NOLETTER)) == 0) return false; - } - free(resp); printf(_("Please answer \"%s\" or \"%s\".\n"), _(PG_YESLETTER), _(PG_NOLETTER)); } diff --git a/src/bin/scripts/createuser.c b/src/bin/scripts/createuser.c index e88879dc19..f13d9a047a 100644 --- a/src/bin/scripts/createuser.c +++ b/src/bin/scripts/createuser.c @@ -66,6 +66,8 @@ main(int argc, char *argv[]) char *conn_limit = NULL; bool pwprompt = false; char *newpassword = NULL; + char newuser_buf[128]; + char newpassword_buf[100]; /* Tri-valued variables. */ enum trivalue createdb = TRI_DEFAULT, @@ -188,7 +190,11 @@ main(int argc, char *argv[]) if (newuser == NULL) { if (interactive) - newuser = simple_prompt("Enter name of role to add: ", 128, true); + { + simple_prompt("Enter name of role to add: ", + newuser_buf, sizeof(newuser_buf), true); + newuser = newuser_buf; + } else { if (getenv("PGUSER")) @@ -200,18 +206,17 @@ main(int argc, char *argv[]) if (pwprompt) { - char *pw1, - *pw2; + char pw2[100]; - pw1 = simple_prompt("Enter password for new role: ", 100, false); - pw2 = simple_prompt("Enter it again: ", 100, false); - if (strcmp(pw1, pw2) != 0) + simple_prompt("Enter password for new role: ", + newpassword_buf, sizeof(newpassword_buf), false); + simple_prompt("Enter it again: ", pw2, sizeof(pw2), false); + if (strcmp(newpassword_buf, pw2) != 0) { fprintf(stderr, _("Passwords didn't match.\n")); exit(1); } - newpassword = pw1; - free(pw2); + newpassword = newpassword_buf; } if (superuser == 0) diff --git a/src/bin/scripts/dropuser.c b/src/bin/scripts/dropuser.c index 31fa28f7cd..ebcc15209c 100644 --- a/src/bin/scripts/dropuser.c +++ b/src/bin/scripts/dropuser.c @@ -46,6 +46,7 @@ main(int argc, char *argv[]) enum trivalue prompt_password = TRI_DEFAULT; bool echo = false; bool interactive = false; + char dropuser_buf[128]; PQExpBufferData sql; @@ -108,7 +109,11 @@ main(int argc, char *argv[]) if (dropuser == NULL) { if (interactive) - dropuser = simple_prompt("Enter name of role to drop: ", 128, true); + { + simple_prompt("Enter name of role to drop: ", + dropuser_buf, sizeof(dropuser_buf), true); + dropuser = dropuser_buf; + } else { fprintf(stderr, _("%s: missing required argument role name\n"), progname); diff --git a/src/include/port.h b/src/include/port.h index 455f72338c..b81fa4a89e 100644 --- a/src/include/port.h +++ b/src/include/port.h @@ -203,7 +203,8 @@ extern char *pgwin32_setlocale(int category, const char *locale); #endif /* WIN32 */ /* Portable prompt handling */ -extern char *simple_prompt(const char *prompt, int maxlen, bool echo); +extern void simple_prompt(const char *prompt, char *destination, size_t destlen, + bool echo); #ifdef WIN32 #define PG_SIGNAL_COUNT 32 diff --git a/src/port/sprompt.c b/src/port/sprompt.c index fd6f16ed30..15ca7a6005 100644 --- a/src/port/sprompt.c +++ b/src/port/sprompt.c @@ -12,33 +12,31 @@ * *------------------------------------------------------------------------- */ +#include "c.h" + +#ifdef HAVE_TERMIOS_H +#include +#endif /* * simple_prompt * * Generalized function especially intended for reading in usernames and - * password interactively. Reads from /dev/tty or stdin/stderr. + * passwords interactively. Reads from /dev/tty or stdin/stderr. * - * prompt: The prompt to print - * maxlen: How many characters to accept + * prompt: The prompt to print, or NULL if none (automatically localized) + * destination: buffer in which to store result + * destlen: allocated length of destination * echo: Set to false if you want to hide what is entered (for passwords) * - * Returns a malloc()'ed string with the input (w/o trailing newline). + * The input (without trailing newline) is returned in the destination buffer, + * with a '\0' appended. */ -#include "c.h" - -#ifdef HAVE_TERMIOS_H -#include -#endif - -extern char *simple_prompt(const char *prompt, int maxlen, bool echo); - -char * -simple_prompt(const char *prompt, int maxlen, bool echo) +void +simple_prompt(const char *prompt, char *destination, size_t destlen, bool echo) { int length; - char *destination; FILE *termin, *termout; @@ -48,14 +46,10 @@ simple_prompt(const char *prompt, int maxlen, bool echo) #else #ifdef WIN32 HANDLE t = NULL; - LPDWORD t_orig = NULL; + DWORD t_orig = 0; #endif #endif - destination = (char *) malloc(maxlen + 1); - if (!destination) - return NULL; - #ifdef WIN32 /* @@ -118,11 +112,10 @@ simple_prompt(const char *prompt, int maxlen, bool echo) if (!echo) { /* get a new handle to turn echo off */ - t_orig = (LPDWORD) malloc(sizeof(DWORD)); t = GetStdHandle(STD_INPUT_HANDLE); /* save the old configuration first */ - GetConsoleMode(t, t_orig); + GetConsoleMode(t, &t_orig); /* set to the new mode */ SetConsoleMode(t, ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT); @@ -136,7 +129,7 @@ simple_prompt(const char *prompt, int maxlen, bool echo) fflush(termout); } - if (fgets(destination, maxlen + 1, termin) == NULL) + if (fgets(destination, destlen, termin) == NULL) destination[0] = '\0'; length = strlen(destination); @@ -170,10 +163,9 @@ simple_prompt(const char *prompt, int maxlen, bool echo) if (!echo) { /* reset to the original console mode */ - SetConsoleMode(t, *t_orig); + SetConsoleMode(t, t_orig); fputs("\n", termout); fflush(termout); - free(t_orig); } #endif #endif @@ -183,6 +175,4 @@ simple_prompt(const char *prompt, int maxlen, bool echo) fclose(termin); fclose(termout); } - - return destination; } From 052cc223d5ce1b727f62afff75797c88d82f880b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 30 Aug 2016 18:22:43 -0400 Subject: [PATCH 088/871] Fix a bunch of places that called malloc and friends with no NULL check. Where possible, use palloc or pg_malloc instead; otherwise, insert explicit NULL checks. Generally speaking, these are places where an actual OOM is quite unlikely, either because they're in client programs that don't allocate all that much, or they're very early in process startup so that we'd likely have had a fork() failure instead. Hence, no back-patch, even though this is nominally a bug fix. Michael Paquier, with some adjustments by me Discussion: --- contrib/pg_standby/pg_standby.c | 2 +- contrib/vacuumlo/vacuumlo.c | 8 ++-- src/backend/bootstrap/bootstrap.c | 14 ++---- src/backend/port/dynloader/darwin.c | 3 ++ src/backend/utils/misc/ps_status.c | 27 +++++++++++ src/bin/pg_archivecleanup/pg_archivecleanup.c | 2 +- src/bin/psql/command.c | 12 ++++- src/common/exec.c | 9 +++- src/test/isolation/isolationtester.c | 14 +++--- src/test/isolation/specparse.y | 34 ++++++------- src/test/isolation/specscanner.l | 4 +- src/test/regress/pg_regress.c | 48 +++++++++---------- 12 files changed, 108 insertions(+), 69 deletions(-) diff --git a/contrib/pg_standby/pg_standby.c b/contrib/pg_standby/pg_standby.c index 5eac2b1e49..e4136f9149 100644 --- a/contrib/pg_standby/pg_standby.c +++ b/contrib/pg_standby/pg_standby.c @@ -632,7 +632,7 @@ main(int argc, char **argv) } break; case 't': /* Trigger file */ - triggerPath = strdup(optarg); + triggerPath = pg_strdup(optarg); break; case 'w': /* Max wait time */ maxwaittime = atoi(optarg); diff --git a/contrib/vacuumlo/vacuumlo.c b/contrib/vacuumlo/vacuumlo.c index 0a9328dc0e..221e1dbcb1 100644 --- a/contrib/vacuumlo/vacuumlo.c +++ b/contrib/vacuumlo/vacuumlo.c @@ -242,7 +242,7 @@ vacuumlo(const char *database, const struct _param * param) if (!schema || !table || !field) { - fprintf(stderr, "Out of memory\n"); + fprintf(stderr, "%s", PQerrorMessage(conn)); PQclear(res); PQfinish(conn); if (schema != NULL) @@ -519,7 +519,7 @@ main(int argc, char **argv) } break; case 'U': - param.pg_user = strdup(optarg); + param.pg_user = pg_strdup(optarg); break; case 'w': param.pg_prompt = TRI_NO; @@ -534,10 +534,10 @@ main(int argc, char **argv) fprintf(stderr, "%s: invalid port number: %s\n", progname, optarg); exit(1); } - param.pg_port = strdup(optarg); + param.pg_port = pg_strdup(optarg); break; case 'h': - param.pg_host = strdup(optarg); + param.pg_host = pg_strdup(optarg); break; } } diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 8feeae05df..3870a4deb9 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -47,7 +47,8 @@ uint32 bootstrap_data_checksum_version = 0; /* No checksum */ -#define ALLOC(t, c) ((t *) calloc((unsigned)(c), sizeof(t))) +#define ALLOC(t, c) \ + ((t *) MemoryContextAllocZero(TopMemoryContext, (unsigned)(c) * sizeof(t))) static void CheckerModeMain(void); static void BootstrapModeMain(void); @@ -227,7 +228,7 @@ AuxiliaryProcessMain(int argc, char *argv[]) SetConfigOption("shared_buffers", optarg, PGC_POSTMASTER, PGC_S_ARGV); break; case 'D': - userDoption = strdup(optarg); + userDoption = pstrdup(optarg); break; case 'd': { @@ -1002,13 +1003,8 @@ boot_get_type_io_data(Oid typid, static Form_pg_attribute AllocateAttribute(void) { - Form_pg_attribute attribute = (Form_pg_attribute) malloc(ATTRIBUTE_FIXED_PART_SIZE); - - if (!PointerIsValid(attribute)) - elog(FATAL, "out of memory"); - MemSet(attribute, 0, ATTRIBUTE_FIXED_PART_SIZE); - - return attribute; + return (Form_pg_attribute) + MemoryContextAllocZero(TopMemoryContext, ATTRIBUTE_FIXED_PART_SIZE); } /* diff --git a/src/backend/port/dynloader/darwin.c b/src/backend/port/dynloader/darwin.c index ccd92c39d4..a83c614f4f 100644 --- a/src/backend/port/dynloader/darwin.c +++ b/src/backend/port/dynloader/darwin.c @@ -78,6 +78,9 @@ pg_dlsym(void *handle, char *funcname) NSSymbol symbol; char *symname = (char *) malloc(strlen(funcname) + 2); + if (!symname) + return NULL; + sprintf(symname, "_%s", funcname); if (NSIsSymbolNameDefined(symname)) { diff --git a/src/backend/utils/misc/ps_status.c b/src/backend/utils/misc/ps_status.c index 892a810bab..c50be8aab6 100644 --- a/src/backend/utils/misc/ps_status.c +++ b/src/backend/utils/misc/ps_status.c @@ -113,6 +113,9 @@ static char **save_argv; * overwritten during init_ps_display. Also, the physical location of the * environment strings may be moved, so this should be called before any code * that might try to hang onto a getenv() result.) + * + * Note that in case of failure this cannot call elog() as that is not + * initialized yet. We rely on write_stderr() instead. */ char ** save_ps_display_args(int argc, char **argv) @@ -163,8 +166,20 @@ save_ps_display_args(int argc, char **argv) * move the environment out of the way */ new_environ = (char **) malloc((i + 1) * sizeof(char *)); + if (!new_environ) + { + write_stderr("out of memory\n"); + exit(1); + } for (i = 0; environ[i] != NULL; i++) + { new_environ[i] = strdup(environ[i]); + if (!new_environ[i]) + { + write_stderr("out of memory\n"); + exit(1); + } + } new_environ[i] = NULL; environ = new_environ; } @@ -189,8 +204,20 @@ save_ps_display_args(int argc, char **argv) int i; new_argv = (char **) malloc((argc + 1) * sizeof(char *)); + if (!new_argv) + { + write_stderr("out of memory\n"); + exit(1); + } for (i = 0; i < argc; i++) + { new_argv[i] = strdup(argv[i]); + if (!new_argv[i]) + { + write_stderr("out of memory\n"); + exit(1); + } + } new_argv[argc] = NULL; #if defined(__darwin__) diff --git a/src/bin/pg_archivecleanup/pg_archivecleanup.c b/src/bin/pg_archivecleanup/pg_archivecleanup.c index 2b3d15dd58..319038fc80 100644 --- a/src/bin/pg_archivecleanup/pg_archivecleanup.c +++ b/src/bin/pg_archivecleanup/pg_archivecleanup.c @@ -314,7 +314,7 @@ main(int argc, char **argv) dryrun = true; break; case 'x': - additional_ext = strdup(optarg); /* Extension to remove + additional_ext = pg_strdup(optarg); /* Extension to remove * from xlogfile names */ break; default: diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 38038d415f..a9a2fdbf26 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -1182,15 +1182,23 @@ exec_command(const char *cmd, fflush(stdout); } result = gets_fromFile(stdin); + if (!result) + { + psql_error("\\%s: could not read value for variable\n", + cmd); + success = false; + } } - if (!SetVariable(pset.vars, opt, result)) + if (result && + !SetVariable(pset.vars, opt, result)) { psql_error("\\%s: error while setting variable\n", cmd); success = false; } - free(result); + if (result) + free(result); if (prompt_text) free(prompt_text); free(opt); diff --git a/src/common/exec.c b/src/common/exec.c index d736b02280..a2de0718cf 100644 --- a/src/common/exec.c +++ b/src/common/exec.c @@ -553,6 +553,7 @@ set_pglocale_pgservice(const char *argv0, const char *app) char my_exec_path[MAXPGPATH]; char env_path[MAXPGPATH + sizeof("PGSYSCONFDIR=")]; /* longer than * PGLOCALEDIR */ + char *dup_path; /* don't set LC_ALL in the backend */ if (strcmp(app, PG_TEXTDOMAIN("postgres")) != 0) @@ -583,7 +584,9 @@ set_pglocale_pgservice(const char *argv0, const char *app) /* set for libpq to use */ snprintf(env_path, sizeof(env_path), "PGLOCALEDIR=%s", path); canonicalize_path(env_path + 12); - putenv(strdup(env_path)); + dup_path = strdup(env_path); + if (dup_path) + putenv(dup_path); } #endif @@ -594,7 +597,9 @@ set_pglocale_pgservice(const char *argv0, const char *app) /* set for libpq to use */ snprintf(env_path, sizeof(env_path), "PGSYSCONFDIR=%s", path); canonicalize_path(env_path + 13); - putenv(strdup(env_path)); + dup_path = strdup(env_path); + if (dup_path) + putenv(dup_path); } } diff --git a/src/test/isolation/isolationtester.c b/src/test/isolation/isolationtester.c index 908a7ce800..db2b55982b 100644 --- a/src/test/isolation/isolationtester.c +++ b/src/test/isolation/isolationtester.c @@ -119,7 +119,7 @@ main(int argc, char **argv) for (i = 0; i < testspec->nsessions; i++) nallsteps += testspec->sessions[i]->nsteps; - allsteps = malloc(nallsteps * sizeof(Step *)); + allsteps = pg_malloc(nallsteps * sizeof(Step *)); n = 0; for (i = 0; i < testspec->nsessions; i++) @@ -190,7 +190,7 @@ main(int argc, char **argv) if (PQresultStatus(res) == PGRES_TUPLES_OK) { if (PQntuples(res) == 1 && PQnfields(res) == 1) - backend_pids[i] = strdup(PQgetvalue(res, 0, 0)); + backend_pids[i] = pg_strdup(PQgetvalue(res, 0, 0)); else { fprintf(stderr, "backend pid query returned %d rows and %d columns, expected 1 row and 1 column", @@ -286,7 +286,7 @@ run_all_permutations(TestSpec *testspec) for (i = 0; i < testspec->nsessions; i++) nsteps += testspec->sessions[i]->nsteps; - steps = malloc(sizeof(Step *) * nsteps); + steps = pg_malloc(sizeof(Step *) * nsteps); /* * To generate the permutations, we conceptually put the steps of each @@ -297,7 +297,7 @@ run_all_permutations(TestSpec *testspec) * A pile is actually just an integer which tells how many steps we've * already picked from this pile. */ - piles = malloc(sizeof(int) * testspec->nsessions); + piles = pg_malloc(sizeof(int) * testspec->nsessions); for (i = 0; i < testspec->nsessions; i++) piles[i] = 0; @@ -345,7 +345,7 @@ run_named_permutations(TestSpec *testspec) Permutation *p = testspec->permutations[i]; Step **steps; - steps = malloc(p->nsteps * sizeof(Step *)); + steps = pg_malloc(p->nsteps * sizeof(Step *)); /* Find all the named steps using the lookup table */ for (j = 0; j < p->nsteps; j++) @@ -476,8 +476,8 @@ run_permutation(TestSpec *testspec, int nsteps, Step **steps) return; } - waiting = malloc(sizeof(Step *) * testspec->nsessions); - errorstep = malloc(sizeof(Step *) * testspec->nsessions); + waiting = pg_malloc(sizeof(Step *) * testspec->nsessions); + errorstep = pg_malloc(sizeof(Step *) * testspec->nsessions); printf("\nstarting permutation:"); for (i = 0; i < nsteps; i++) diff --git a/src/test/isolation/specparse.y b/src/test/isolation/specparse.y index fce6cc6c94..59744cea54 100644 --- a/src/test/isolation/specparse.y +++ b/src/test/isolation/specparse.y @@ -73,8 +73,8 @@ setup_list: } | setup_list setup { - $$.elements = realloc($1.elements, - ($1.nelements + 1) * sizeof(void *)); + $$.elements = pg_realloc($1.elements, + ($1.nelements + 1) * sizeof(void *)); $$.elements[$1.nelements] = $2; $$.nelements = $1.nelements + 1; } @@ -97,15 +97,15 @@ opt_teardown: session_list: session_list session { - $$.elements = realloc($1.elements, - ($1.nelements + 1) * sizeof(void *)); + $$.elements = pg_realloc($1.elements, + ($1.nelements + 1) * sizeof(void *)); $$.elements[$1.nelements] = $2; $$.nelements = $1.nelements + 1; } | session { $$.nelements = 1; - $$.elements = malloc(sizeof(void *)); + $$.elements = pg_malloc(sizeof(void *)); $$.elements[0] = $1; } ; @@ -113,7 +113,7 @@ session_list: session: SESSION string_literal opt_setup step_list opt_teardown { - $$ = malloc(sizeof(Session)); + $$ = pg_malloc(sizeof(Session)); $$->name = $2; $$->setupsql = $3; $$->steps = (Step **) $4.elements; @@ -125,15 +125,15 @@ session: step_list: step_list step { - $$.elements = realloc($1.elements, - ($1.nelements + 1) * sizeof(void *)); + $$.elements = pg_realloc($1.elements, + ($1.nelements + 1) * sizeof(void *)); $$.elements[$1.nelements] = $2; $$.nelements = $1.nelements + 1; } | step { $$.nelements = 1; - $$.elements = malloc(sizeof(void *)); + $$.elements = pg_malloc(sizeof(void *)); $$.elements[0] = $1; } ; @@ -142,7 +142,7 @@ step_list: step: STEP string_literal sqlblock { - $$ = malloc(sizeof(Step)); + $$ = pg_malloc(sizeof(Step)); $$->name = $2; $$->sql = $3; $$->errormsg = NULL; @@ -164,15 +164,15 @@ opt_permutation_list: permutation_list: permutation_list permutation { - $$.elements = realloc($1.elements, - ($1.nelements + 1) * sizeof(void *)); + $$.elements = pg_realloc($1.elements, + ($1.nelements + 1) * sizeof(void *)); $$.elements[$1.nelements] = $2; $$.nelements = $1.nelements + 1; } | permutation { $$.nelements = 1; - $$.elements = malloc(sizeof(void *)); + $$.elements = pg_malloc(sizeof(void *)); $$.elements[0] = $1; } ; @@ -181,7 +181,7 @@ permutation_list: permutation: PERMUTATION string_literal_list { - $$ = malloc(sizeof(Permutation)); + $$ = pg_malloc(sizeof(Permutation)); $$->stepnames = (char **) $2.elements; $$->nsteps = $2.nelements; } @@ -190,15 +190,15 @@ permutation: string_literal_list: string_literal_list string_literal { - $$.elements = realloc($1.elements, - ($1.nelements + 1) * sizeof(void *)); + $$.elements = pg_realloc($1.elements, + ($1.nelements + 1) * sizeof(void *)); $$.elements[$1.nelements] = $2; $$.nelements = $1.nelements + 1; } | string_literal { $$.nelements = 1; - $$.elements = malloc(sizeof(void *)); + $$.elements = pg_malloc(sizeof(void *)); $$.elements[0] = $1; } ; diff --git a/src/test/isolation/specscanner.l b/src/test/isolation/specscanner.l index 675bd21f17..6f081d567c 100644 --- a/src/test/isolation/specscanner.l +++ b/src/test/isolation/specscanner.l @@ -56,7 +56,7 @@ teardown { return(TEARDOWN); } } \" { litbuf[litbufpos] = '\0'; - yylval.str = strdup(litbuf); + yylval.str = pg_strdup(litbuf); BEGIN(INITIAL); return(string_literal); } @@ -72,7 +72,7 @@ teardown { return(TEARDOWN); } } {space}*"}" { litbuf[litbufpos] = '\0'; - yylval.str = strdup(litbuf); + yylval.str = pg_strdup(litbuf); BEGIN(INITIAL); return(sqlblock); } diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 574f5b87be..2260057840 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -151,10 +151,10 @@ unlimit_core_size(void) void add_stringlist_item(_stringlist **listhead, const char *str) { - _stringlist *newentry = malloc(sizeof(_stringlist)); + _stringlist *newentry = pg_malloc(sizeof(_stringlist)); _stringlist *oldentry; - newentry->str = strdup(str); + newentry->str = pg_strdup(str); newentry->next = NULL; if (*listhead == NULL) *listhead = newentry; @@ -187,7 +187,7 @@ free_stringlist(_stringlist **listhead) static void split_to_stringlist(const char *s, const char *delim, _stringlist **listhead) { - char *sc = strdup(s); + char *sc = pg_strdup(s); char *token = strtok(sc, delim); while (token) @@ -327,7 +327,7 @@ signal_remove_temp(int signum) static const char * make_temp_sockdir(void) { - char *template = strdup("/tmp/pg_regress-XXXXXX"); + char *template = pg_strdup("/tmp/pg_regress-XXXXXX"); temp_sockdir = mkdtemp(template); if (temp_sockdir == NULL) @@ -441,7 +441,7 @@ replace_string(char *string, char *replace, char *replacement) while ((ptr = strstr(string, replace)) != NULL) { - char *dup = strdup(string); + char *dup = pg_strdup(string); strlcpy(string, dup, ptr - string + 1); strcat(string, replacement); @@ -661,11 +661,11 @@ load_resultmap(void) */ if (string_matches_pattern(host_platform, platform)) { - _resultmap *entry = malloc(sizeof(_resultmap)); + _resultmap *entry = pg_malloc(sizeof(_resultmap)); - entry->test = strdup(buf); - entry->type = strdup(file_type); - entry->resultfile = strdup(expected); + entry->test = pg_strdup(buf); + entry->type = pg_strdup(file_type); + entry->resultfile = pg_strdup(expected); entry->next = resultmap; resultmap = entry; } @@ -908,7 +908,7 @@ current_windows_user(const char **acct, const char **dom) progname, GetLastError()); exit(2); } - tokenuser = malloc(retlen); + tokenuser = pg_malloc(retlen); if (!GetTokenInformation(token, TokenUser, tokenuser, retlen, &retlen)) { fprintf(stderr, @@ -1460,7 +1460,7 @@ wait_for_tests(PID_TYPE * pids, int *statuses, char **names, int num_tests) int i; #ifdef WIN32 - PID_TYPE *active_pids = malloc(num_tests * sizeof(PID_TYPE)); + PID_TYPE *active_pids = pg_malloc(num_tests * sizeof(PID_TYPE)); memcpy(active_pids, pids, num_tests * sizeof(PID_TYPE)); #endif @@ -1848,7 +1848,7 @@ open_result_files(void) /* create the log file (copy of running status output) */ snprintf(file, sizeof(file), "%s/regression.out", outputdir); - logfilename = strdup(file); + logfilename = pg_strdup(file); logfile = fopen(logfilename, "w"); if (!logfile) { @@ -1859,7 +1859,7 @@ open_result_files(void) /* create the diffs file as empty */ snprintf(file, sizeof(file), "%s/regression.diffs", outputdir); - difffilename = strdup(file); + difffilename = pg_strdup(file); difffile = fopen(difffilename, "w"); if (!difffile) { @@ -2064,13 +2064,13 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc * before we add the specified one. */ free_stringlist(&dblist); - split_to_stringlist(strdup(optarg), ", ", &dblist); + split_to_stringlist(pg_strdup(optarg), ", ", &dblist); break; case 2: debug = true; break; case 3: - inputdir = strdup(optarg); + inputdir = pg_strdup(optarg); break; case 4: add_stringlist_item(&loadlanguage, optarg); @@ -2079,10 +2079,10 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc max_connections = atoi(optarg); break; case 6: - encoding = strdup(optarg); + encoding = pg_strdup(optarg); break; case 7: - outputdir = strdup(optarg); + outputdir = pg_strdup(optarg); break; case 8: add_stringlist_item(&schedulelist, optarg); @@ -2094,27 +2094,27 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc nolocale = true; break; case 13: - hostname = strdup(optarg); + hostname = pg_strdup(optarg); break; case 14: port = atoi(optarg); port_specified_by_user = true; break; case 15: - user = strdup(optarg); + user = pg_strdup(optarg); break; case 16: /* "--bindir=" means to use PATH */ if (strlen(optarg)) - bindir = strdup(optarg); + bindir = pg_strdup(optarg); else bindir = NULL; break; case 17: - dlpath = strdup(optarg); + dlpath = pg_strdup(optarg); break; case 18: - split_to_stringlist(strdup(optarg), ", ", &extraroles); + split_to_stringlist(pg_strdup(optarg), ", ", &extraroles); break; case 19: add_stringlist_item(&temp_configs, optarg); @@ -2123,13 +2123,13 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc use_existing = true; break; case 21: - launcher = strdup(optarg); + launcher = pg_strdup(optarg); break; case 22: add_stringlist_item(&loadextension, optarg); break; case 24: - config_auth_datadir = pstrdup(optarg); + config_auth_datadir = pg_strdup(optarg); break; default: /* getopt_long already emitted a complaint */ From 530fb68e0f10ba921922e7b88403fcd2bd263742 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 31 Aug 2016 12:36:18 +0530 Subject: [PATCH 089/871] Update comments to reflect code rearrangement. Commit f9143d102ffd0947ca904c62b1d3d6fd587e0c80 falsified these. KaiGai Kohei --- src/include/nodes/execnodes.h | 3 ++- src/include/nodes/plannodes.h | 3 ++- src/include/nodes/relation.h | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index e7fd7bd08e..a4ea1b901a 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1611,7 +1611,8 @@ struct CustomExecMethods; typedef struct CustomScanState { ScanState ss; - uint32 flags; /* mask of CUSTOMPATH_* flags, see relation.h */ + uint32 flags; /* mask of CUSTOMPATH_* flags, see + * nodes/extensible.h */ List *custom_ps; /* list of child PlanState nodes, if any */ Size pscan_len; /* size of parallel coordination information */ const struct CustomExecMethods *methods; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index bc5463b8f7..e2fbc7d5a7 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -560,7 +560,8 @@ struct CustomScanMethods; typedef struct CustomScan { Scan scan; - uint32 flags; /* mask of CUSTOMPATH_* flags, see relation.h */ + uint32 flags; /* mask of CUSTOMPATH_* flags, see + * nodes/extensible.h */ List *custom_plans; /* list of Plan nodes, if any */ List *custom_exprs; /* expressions that custom code may evaluate */ List *custom_private; /* private data for custom code */ diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index fcfb0d4d0f..2709cc7df5 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -1087,7 +1087,8 @@ struct CustomPathMethods; typedef struct CustomPath { Path path; - uint32 flags; /* mask of CUSTOMPATH_* flags, see above */ + uint32 flags; /* mask of CUSTOMPATH_* flags, see + * nodes/extensible.h */ List *custom_paths; /* list of child Path nodes, if any */ List *custom_private; const struct CustomPathMethods *methods; From 0e0f43d6fdc2e1fbd5261245ed4cf85302a3f653 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 31 Aug 2016 08:52:13 -0400 Subject: [PATCH 090/871] Prevent starting a standalone backend with standby_mode on. This can't really work because standby_mode expects there to be more WAL arriving, which there will not ever be because there's no WAL receiver process to fetch it. Moreover, if standby_mode is on then hot standby might also be turned on, causing even more strangeness because that expects read-only sessions to be executing in parallel. Bernd Helmle reported a case where btree_xlog_delete_get_latestRemovedXid got confused, but rather than band-aiding individual problems it seems best to prevent getting anywhere near this state in the first place. Back-patch to all supported branches. In passing, also fix some omissions of errcodes in other ereport's in readRecoveryCommandFile(). Michael Paquier (errcode hacking by me) Discussion: <00F0B2CEF6D0CEF8A90119D4@eje.credativ.lan> --- src/backend/access/transam/xlog.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index acd95aa740..0b991bb91d 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5022,7 +5022,8 @@ readRecoveryCommandFile(void) rtli = (TimeLineID) strtoul(item->value, NULL, 0); if (errno == EINVAL || errno == ERANGE) ereport(FATAL, - (errmsg("recovery_target_timeline is not a valid number: \"%s\"", + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("recovery_target_timeline is not a valid number: \"%s\"", item->value))); } if (rtli) @@ -5038,7 +5039,8 @@ readRecoveryCommandFile(void) recoveryTargetXid = (TransactionId) strtoul(item->value, NULL, 0); if (errno == EINVAL || errno == ERANGE) ereport(FATAL, - (errmsg("recovery_target_xid is not a valid number: \"%s\"", + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("recovery_target_xid is not a valid number: \"%s\"", item->value))); ereport(DEBUG2, (errmsg_internal("recovery_target_xid = %u", @@ -5153,7 +5155,8 @@ readRecoveryCommandFile(void) } else ereport(FATAL, - (errmsg("unrecognized recovery parameter \"%s\"", + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unrecognized recovery parameter \"%s\"", item->name))); } @@ -5172,7 +5175,8 @@ readRecoveryCommandFile(void) { if (recoveryRestoreCommand == NULL) ereport(FATAL, - (errmsg("recovery command file \"%s\" must specify restore_command when standby mode is not enabled", + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("recovery command file \"%s\" must specify restore_command when standby mode is not enabled", RECOVERY_COMMAND_FILE))); } @@ -5186,6 +5190,15 @@ readRecoveryCommandFile(void) !EnableHotStandby) recoveryTargetAction = RECOVERY_TARGET_ACTION_SHUTDOWN; + /* + * We don't support standby_mode in standalone backends; that requires + * other processes such as the WAL receiver to be alive. + */ + if (StandbyModeRequested && !IsUnderPostmaster) + ereport(FATAL, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("standby mode is not supported by single-user servers"))); + /* Enable fetching from archive recovery area */ ArchiveRecoveryRequested = true; @@ -5202,7 +5215,8 @@ readRecoveryCommandFile(void) /* Timeline 1 does not have a history file, all else should */ if (rtli != 1 && !existsTimeLineHistory(rtli)) ereport(FATAL, - (errmsg("recovery target timeline %u does not exist", + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("recovery target timeline %u does not exist", rtli))); recoveryTargetTLI = rtli; recoveryTargetIsLatest = false; From 14cca1bf8e31ed39dbc26dd6c610f1113e759972 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 31 Aug 2016 16:00:28 +0300 Subject: [PATCH 091/871] Use static inline functions for float <-> Datum conversions. Now that we are OK with using static inline functions, we can use them to avoid function call overhead of pass-by-val versions of Float4GetDatum, DatumGetFloat8, and Float8GetDatum. Those functions are only a few CPU instructions long, but they could not be written into macros previously, because we need a local union variable for the conversion. I kept the pass-by-ref versions as regular functions. They are very simple too, but they call palloc() anyway, so shaving a few instructions from the function call doesn't seem so important there. Discussion: --- src/backend/utils/fmgr/fmgr.c | 63 ++++------------------------------- src/include/postgres.h | 63 +++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 59 deletions(-) diff --git a/src/backend/utils/fmgr/fmgr.c b/src/backend/utils/fmgr/fmgr.c index 7e6a60d624..7aae35074f 100644 --- a/src/backend/utils/fmgr/fmgr.c +++ b/src/backend/utils/fmgr/fmgr.c @@ -2126,10 +2126,7 @@ fmgr(Oid procedureId,...) * * int8, float4, and float8 can be passed by value if Datum is wide enough. * (For backwards-compatibility reasons, we allow pass-by-ref to be chosen - * at compile time even if pass-by-val is possible.) For the float types, - * we need a support routine even if we are passing by value, because many - * machines pass int and float function parameters/results differently; - * so we need to play weird games with unions. + * at compile time even if pass-by-val is possible.) * * Note: there is only one switch controlling the pass-by-value option for * both int8 and float8; this is to avoid making things unduly complicated @@ -2149,77 +2146,29 @@ Int64GetDatum(int64 X) } #endif /* USE_FLOAT8_BYVAL */ +#ifndef USE_FLOAT4_BYVAL + Datum Float4GetDatum(float4 X) { -#ifdef USE_FLOAT4_BYVAL - union - { - float4 value; - int32 retval; - } myunion; - - myunion.value = X; - return SET_4_BYTES(myunion.retval); -#else float4 *retval = (float4 *) palloc(sizeof(float4)); *retval = X; return PointerGetDatum(retval); -#endif } +#endif -#ifdef USE_FLOAT4_BYVAL - -float4 -DatumGetFloat4(Datum X) -{ - union - { - int32 value; - float4 retval; - } myunion; - - myunion.value = GET_4_BYTES(X); - return myunion.retval; -} -#endif /* USE_FLOAT4_BYVAL */ +#ifndef USE_FLOAT8_BYVAL Datum Float8GetDatum(float8 X) { -#ifdef USE_FLOAT8_BYVAL - union - { - float8 value; - int64 retval; - } myunion; - - myunion.value = X; - return SET_8_BYTES(myunion.retval); -#else float8 *retval = (float8 *) palloc(sizeof(float8)); *retval = X; return PointerGetDatum(retval); -#endif } - -#ifdef USE_FLOAT8_BYVAL - -float8 -DatumGetFloat8(Datum X) -{ - union - { - int64 value; - float8 retval; - } myunion; - - myunion.value = GET_8_BYTES(X); - return myunion.retval; -} -#endif /* USE_FLOAT8_BYVAL */ +#endif /*------------------------------------------------------------------------- diff --git a/src/include/postgres.h b/src/include/postgres.h index fb1933f8f2..d999013238 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -656,6 +656,14 @@ extern Datum Int64GetDatum(int64 X); #define UInt64GetDatum(X) Int64GetDatum((int64) (X)) #endif +/* + * Float <-> Datum conversions + * + * These have to be implemented as inline functions rather than macros, when + * passing by value, because many machines pass int and float function + * parameters/results differently; so we need to play weird games with unions. + */ + /* * DatumGetFloat4 * Returns 4-byte floating point value of a datum. @@ -664,7 +672,18 @@ extern Datum Int64GetDatum(int64 X); */ #ifdef USE_FLOAT4_BYVAL -extern float4 DatumGetFloat4(Datum X); +static inline float4 +DatumGetFloat4(Datum X) +{ + union + { + int32 value; + float4 retval; + } myunion; + + myunion.value = GET_4_BYTES(X); + return myunion.retval; +} #else #define DatumGetFloat4(X) (* ((float4 *) DatumGetPointer(X))) #endif @@ -676,8 +695,22 @@ extern float4 DatumGetFloat4(Datum X); * Note: if float4 is pass by reference, this function returns a reference * to palloc'd space. */ +#ifdef USE_FLOAT4_BYVAL +static inline Datum +Float4GetDatum(float4 X) +{ + union + { + float4 value; + int32 retval; + } myunion; + myunion.value = X; + return SET_4_BYTES(myunion.retval); +} +#else extern Datum Float4GetDatum(float4 X); +#endif /* * DatumGetFloat8 @@ -687,7 +720,18 @@ extern Datum Float4GetDatum(float4 X); */ #ifdef USE_FLOAT8_BYVAL -extern float8 DatumGetFloat8(Datum X); +static inline float8 +DatumGetFloat8(Datum X) +{ + union + { + int64 value; + float8 retval; + } myunion; + + myunion.value = GET_8_BYTES(X); + return myunion.retval; +} #else #define DatumGetFloat8(X) (* ((float8 *) DatumGetPointer(X))) #endif @@ -700,7 +744,22 @@ extern float8 DatumGetFloat8(Datum X); * to palloc'd space. */ +#ifdef USE_FLOAT8_BYVAL +static inline Datum +Float8GetDatum(float8 X) +{ + union + { + float8 value; + int64 retval; + } myunion; + + myunion.value = X; + return SET_8_BYTES(myunion.retval); +} +#else extern Datum Float8GetDatum(float8 X); +#endif /* From 679226337ac3f44e784de0a95a8599dfd86401e8 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 31 Aug 2016 09:24:19 -0400 Subject: [PATCH 092/871] Remove no-longer-useful SSL-specific Port.count field. Since we removed SSL renegotiation, there's no longer any reason to keep track of the amount of data transferred over the link. Daniel Gustafsson Discussion: --- src/backend/libpq/be-secure-openssl.c | 6 ++---- src/include/libpq/libpq-be.h | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index e5f434ca17..bb0d2d977f 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -447,8 +447,6 @@ be_tls_open_server(Port *port) return -1; } - port->count = 0; - /* Get client certificate, if available. */ port->peer = SSL_get_peer_certificate(port->ssl); @@ -549,7 +547,7 @@ be_tls_read(Port *port, void *ptr, size_t len, int *waitfor) switch (err) { case SSL_ERROR_NONE: - port->count += n; + /* a-ok */ break; case SSL_ERROR_WANT_READ: *waitfor = WL_SOCKET_READABLE; @@ -609,7 +607,7 @@ be_tls_write(Port *port, void *ptr, size_t len, int *waitfor) switch (err) { case SSL_ERROR_NONE: - port->count += n; + /* a-ok */ break; case SSL_ERROR_WANT_READ: *waitfor = WL_SOCKET_READABLE; diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index 5d07b78223..ecdfbc60bb 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -192,7 +192,6 @@ typedef struct Port #ifdef USE_OPENSSL SSL *ssl; X509 *peer; - unsigned long count; #endif } Port; From 65a588b4c3b06aefe0b00d6222598d4cff3542c2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 31 Aug 2016 13:58:01 -0400 Subject: [PATCH 093/871] Try to fix portability issue in enum renumbering (again). The hack embodied in commit 4ba61a487 no longer works after today's change to allow DatumGetFloat4/Float4GetDatum to be inlined (commit 14cca1bf8). Probably what's happening is that the faulty compilers are deciding that the now-inlined assignment is a no-op and so they're not required to round to float4 width. We had a bunch of similar issues earlier this year in the degree-based trig functions, and eventually settled on using volatile intermediate variables as the least ugly method of forcing recalcitrant compilers to do what the C standard says (cf commit 82311bcdd). Let's see if that method works here. Discussion: <4640.1472664476@sss.pgh.pa.us> --- src/backend/catalog/pg_enum.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/backend/catalog/pg_enum.c b/src/backend/catalog/pg_enum.c index af89daa712..c66f9632c2 100644 --- a/src/backend/catalog/pg_enum.c +++ b/src/backend/catalog/pg_enum.c @@ -315,21 +315,21 @@ AddEnumLabel(Oid enumTypeOid, newelemorder = nbr_en->enumsortorder + 1; else { - other_nbr_en = (Form_pg_enum) GETSTRUCT(existing[other_nbr_index]); - newelemorder = (nbr_en->enumsortorder + - other_nbr_en->enumsortorder) / 2; - /* - * On some machines, newelemorder may be in a register that's - * wider than float4. We need to force it to be rounded to float4 - * precision before making the following comparisons, or we'll get - * wrong results. (Such behavior violates the C standard, but - * fixing the compilers is out of our reach.) + * The midpoint value computed here has to be rounded to float4 + * precision, else our equality comparisons against the adjacent + * values are meaningless. The most portable way of forcing that + * to happen with non-C-standard-compliant compilers is to store + * it into a volatile variable. */ - newelemorder = DatumGetFloat4(Float4GetDatum(newelemorder)); + volatile float4 midpoint; - if (newelemorder == nbr_en->enumsortorder || - newelemorder == other_nbr_en->enumsortorder) + other_nbr_en = (Form_pg_enum) GETSTRUCT(existing[other_nbr_index]); + midpoint = (nbr_en->enumsortorder + + other_nbr_en->enumsortorder) / 2; + + if (midpoint == nbr_en->enumsortorder || + midpoint == other_nbr_en->enumsortorder) { RenumberEnumType(pg_enum, existing, nelems); /* Clean up and start over */ @@ -337,6 +337,8 @@ AddEnumLabel(Oid enumTypeOid, ReleaseCatCacheList(list); goto restart; } + + newelemorder = midpoint; } } From d062245b5bd591edf6f78bab8d6b8bb3ff69c7a6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 31 Aug 2016 17:27:09 -0400 Subject: [PATCH 094/871] Improve memory management for PL/Tcl functions. Formerly, the memory used to represent a PL/Tcl function was allocated with malloc() or in TopMemoryContext, and we'd leak it all if the function got redefined during the session. Instead, create a per-function context and keep everything in or under that context. Add a reference-counting mechanism (like the one plpgsql has long had) so that we can safely clean up an old function definition, either immediately if it's not being executed or at the end of the outermost execution. Currently, we only detect that a cached function is obsolete when we next attempt to call that function. So this covers the updated-definition case but leaves cruft around after DROP FUNCTION. It's not clear whether it's worth installing a syscache invalidation callback to watch for drops; none of the other PLs do, so for now we won't do it here either. Michael Paquier and Tom Lane Discussion: --- src/pl/tcl/pltcl.c | 260 +++++++++++++++++++++++++-------------------- 1 file changed, 144 insertions(+), 116 deletions(-) diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 2a335aa219..d236890490 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -114,21 +114,33 @@ typedef struct pltcl_interp_desc /********************************************************************** * The information we cache about loaded procedures + * + * The pltcl_proc_desc struct itself, as well as all subsidiary data, + * is stored in the memory context identified by the fn_cxt field. + * We can reclaim all the data by deleting that context, and should do so + * when the fn_refcount goes to zero. (But note that we do not bother + * trying to clean up Tcl's copy of the procedure definition: it's Tcl's + * problem to manage its memory when we replace a proc definition. We do + * not clean up pltcl_proc_descs when a pg_proc row is deleted, only when + * it is updated, and the same policy applies to Tcl's copy as well.) **********************************************************************/ typedef struct pltcl_proc_desc { - char *user_proname; - char *internal_proname; - TransactionId fn_xmin; - ItemPointerData fn_tid; - bool fn_readonly; - bool lanpltrusted; - pltcl_interp_desc *interp_desc; - FmgrInfo result_in_func; - Oid result_typioparam; - int nargs; - FmgrInfo arg_out_func[FUNC_MAX_ARGS]; - bool arg_is_rowtype[FUNC_MAX_ARGS]; + char *user_proname; /* user's name (from pg_proc.proname) */ + char *internal_proname; /* Tcl name (based on function OID) */ + MemoryContext fn_cxt; /* memory context for this procedure */ + unsigned long fn_refcount; /* number of active references */ + TransactionId fn_xmin; /* xmin of pg_proc row */ + ItemPointerData fn_tid; /* TID of pg_proc row */ + bool fn_readonly; /* is function readonly? */ + bool lanpltrusted; /* is it pltcl (vs. pltclu)? */ + pltcl_interp_desc *interp_desc; /* interpreter to use */ + FmgrInfo result_in_func; /* input function for fn's result type */ + Oid result_typioparam; /* param to pass to same */ + int nargs; /* number of arguments */ + /* these arrays have nargs entries: */ + FmgrInfo *arg_out_func; /* output fns for arg types */ + bool *arg_is_rowtype; /* is each arg composite? */ } pltcl_proc_desc; @@ -312,23 +324,6 @@ pltcl_WaitForEvent(CONST86 Tcl_Time *timePtr) } -/* - * This routine is a crock, and so is everyplace that calls it. The problem - * is that the cached form of pltcl functions/queries is allocated permanently - * (mostly via malloc()) and never released until backend exit. Subsidiary - * data structures such as fmgr info records therefore must live forever - * as well. A better implementation would store all this stuff in a per- - * function memory context that could be reclaimed at need. In the meantime, - * fmgr_info_cxt must be called specifying TopMemoryContext so that whatever - * it might allocate, and whatever the eventual function might allocate using - * fn_mcxt, will live forever too. - */ -static void -perm_fmgr_info(Oid functionId, FmgrInfo *finfo) -{ - fmgr_info_cxt(functionId, finfo, TopMemoryContext); -} - /* * _PG_init() - library load-time initialization * @@ -636,6 +631,7 @@ pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted) Datum retval; FunctionCallInfo save_fcinfo; pltcl_proc_desc *save_prodesc; + pltcl_proc_desc *this_prodesc; /* * Ensure that static pointers are saved/restored properly @@ -643,6 +639,16 @@ pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted) save_fcinfo = pltcl_current_fcinfo; save_prodesc = pltcl_current_prodesc; + /* + * Reset pltcl_current_prodesc to null. Anything that sets it non-null + * should increase the prodesc's fn_refcount at the same time. We'll + * decrease the refcount, and then delete the prodesc if it's no longer + * referenced, on the way out of this function. This ensures that + * prodescs live as long as needed even if somebody replaces the + * originating pg_proc row while they're executing. + */ + pltcl_current_prodesc = NULL; + PG_TRY(); { /* @@ -668,14 +674,31 @@ pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted) } PG_CATCH(); { + /* Restore globals, then clean up the prodesc refcount if any */ + this_prodesc = pltcl_current_prodesc; pltcl_current_fcinfo = save_fcinfo; pltcl_current_prodesc = save_prodesc; + if (this_prodesc != NULL) + { + Assert(this_prodesc->fn_refcount > 0); + if (--this_prodesc->fn_refcount == 0) + MemoryContextDelete(this_prodesc->fn_cxt); + } PG_RE_THROW(); } PG_END_TRY(); + /* Restore globals, then clean up the prodesc refcount if any */ + /* (We're being paranoid in case an error is thrown in context deletion) */ + this_prodesc = pltcl_current_prodesc; pltcl_current_fcinfo = save_fcinfo; pltcl_current_prodesc = save_prodesc; + if (this_prodesc != NULL) + { + Assert(this_prodesc->fn_refcount > 0); + if (--this_prodesc->fn_refcount == 0) + MemoryContextDelete(this_prodesc->fn_cxt); + } return retval; } @@ -703,6 +726,7 @@ pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted) false, pltrusted); pltcl_current_prodesc = prodesc; + prodesc->fn_refcount++; interp = prodesc->interp_desc->interp; @@ -864,6 +888,7 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) pltrusted); pltcl_current_prodesc = prodesc; + prodesc->fn_refcount++; interp = prodesc->interp_desc->interp; @@ -1180,6 +1205,7 @@ pltcl_event_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) InvalidOid, true, pltrusted); pltcl_current_prodesc = prodesc; + prodesc->fn_refcount++; interp = prodesc->interp_desc->interp; @@ -1249,6 +1275,10 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, pltcl_proc_ptr *proc_ptr; bool found; pltcl_proc_desc *prodesc; + pltcl_proc_desc *old_prodesc; + volatile MemoryContext proc_cxt = NULL; + Tcl_DString proc_internal_def; + Tcl_DString proc_internal_body; /* We'll need the pg_proc tuple in any case... */ procTup = SearchSysCache1(PROCOID, ObjectIdGetDatum(fn_oid)); @@ -1256,7 +1286,10 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, elog(ERROR, "cache lookup failed for function %u", fn_oid); procStruct = (Form_pg_proc) GETSTRUCT(procTup); - /* Try to find function in pltcl_proc_htab */ + /* + * Look up function in pltcl_proc_htab; if it's not there, create an entry + * and set the entry's proc_ptr to NULL. + */ proc_key.proc_id = fn_oid; proc_key.is_trigger = OidIsValid(tgreloid); proc_key.user_id = pltrusted ? GetUserId() : InvalidOid; @@ -1274,18 +1307,13 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, * This is needed because CREATE OR REPLACE FUNCTION can modify the * function's pg_proc entry without changing its OID. ************************************************************/ - if (prodesc != NULL) + if (prodesc != NULL && + prodesc->fn_xmin == HeapTupleHeaderGetRawXmin(procTup->t_data) && + ItemPointerEquals(&prodesc->fn_tid, &procTup->t_self)) { - bool uptodate; - - uptodate = (prodesc->fn_xmin == HeapTupleHeaderGetRawXmin(procTup->t_data) && - ItemPointerEquals(&prodesc->fn_tid, &procTup->t_self)); - - if (!uptodate) - { - proc_ptr->proc_ptr = NULL; - prodesc = NULL; - } + /* It's still up-to-date, so we can use it */ + ReleaseSysCache(procTup); + return prodesc; } /************************************************************ @@ -1296,14 +1324,14 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, * * Then we load the procedure into the Tcl interpreter. ************************************************************/ - if (prodesc == NULL) + Tcl_DStringInit(&proc_internal_def); + Tcl_DStringInit(&proc_internal_body); + PG_TRY(); { bool is_trigger = OidIsValid(tgreloid); char internal_proname[128]; HeapTuple typeTup; Form_pg_type typeStruct; - Tcl_DString proc_internal_def; - Tcl_DString proc_internal_body; char proc_internal_args[33 * FUNC_MAX_ARGS]; Datum prosrcdatum; bool isnull; @@ -1312,39 +1340,47 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, Tcl_Interp *interp; int i; int tcl_rc; + MemoryContext oldcontext; /************************************************************ * Build our internal proc name from the function's Oid. Append * "_trigger" when appropriate to ensure the normal and trigger * cases are kept separate. Note name must be all-ASCII. ************************************************************/ - if (!is_trigger && !is_event_trigger) - snprintf(internal_proname, sizeof(internal_proname), - "__PLTcl_proc_%u", fn_oid); - else if (is_event_trigger) + if (is_event_trigger) snprintf(internal_proname, sizeof(internal_proname), "__PLTcl_proc_%u_evttrigger", fn_oid); else if (is_trigger) snprintf(internal_proname, sizeof(internal_proname), "__PLTcl_proc_%u_trigger", fn_oid); + else + snprintf(internal_proname, sizeof(internal_proname), + "__PLTcl_proc_%u", fn_oid); /************************************************************ - * Allocate a new procedure description block + * Allocate a context that will hold all PG data for the procedure. + * We use the internal proc name as the context name. ************************************************************/ - prodesc = (pltcl_proc_desc *) malloc(sizeof(pltcl_proc_desc)); - if (prodesc == NULL) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"))); - MemSet(prodesc, 0, sizeof(pltcl_proc_desc)); - prodesc->user_proname = strdup(NameStr(procStruct->proname)); - prodesc->internal_proname = strdup(internal_proname); - if (prodesc->user_proname == NULL || prodesc->internal_proname == NULL) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"))); + proc_cxt = AllocSetContextCreate(TopMemoryContext, + internal_proname, + ALLOCSET_SMALL_SIZES); + + /************************************************************ + * Allocate and fill a new procedure description block. + * struct prodesc and subsidiary data must all live in proc_cxt. + ************************************************************/ + oldcontext = MemoryContextSwitchTo(proc_cxt); + prodesc = (pltcl_proc_desc *) palloc0(sizeof(pltcl_proc_desc)); + prodesc->user_proname = pstrdup(NameStr(procStruct->proname)); + prodesc->internal_proname = pstrdup(internal_proname); + prodesc->fn_cxt = proc_cxt; + prodesc->fn_refcount = 0; prodesc->fn_xmin = HeapTupleHeaderGetRawXmin(procTup->t_data); prodesc->fn_tid = procTup->t_self; + prodesc->nargs = procStruct->pronargs; + prodesc->arg_out_func = (FmgrInfo *) palloc0(prodesc->nargs * sizeof(FmgrInfo)); + prodesc->arg_is_rowtype = (bool *) palloc0(prodesc->nargs * sizeof(bool)); + MemoryContextSwitchTo(oldcontext); /* Remember if function is STABLE/IMMUTABLE */ prodesc->fn_readonly = @@ -1368,13 +1404,8 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, SearchSysCache1(TYPEOID, ObjectIdGetDatum(procStruct->prorettype)); if (!HeapTupleIsValid(typeTup)) - { - free(prodesc->user_proname); - free(prodesc->internal_proname); - free(prodesc); elog(ERROR, "cache lookup failed for type %u", procStruct->prorettype); - } typeStruct = (Form_pg_type) GETSTRUCT(typeTup); /* Disallow pseudotype result, except VOID */ @@ -1384,37 +1415,24 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, /* okay */ ; else if (procStruct->prorettype == TRIGGEROID || procStruct->prorettype == EVTTRIGGEROID) - { - free(prodesc->user_proname); - free(prodesc->internal_proname); - free(prodesc); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("trigger functions can only be called as triggers"))); - } else - { - free(prodesc->user_proname); - free(prodesc->internal_proname); - free(prodesc); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("PL/Tcl functions cannot return type %s", format_type_be(procStruct->prorettype)))); - } } if (typeStruct->typtype == TYPTYPE_COMPOSITE) - { - free(prodesc->user_proname); - free(prodesc->internal_proname); - free(prodesc); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("PL/Tcl functions cannot return composite types"))); - } - perm_fmgr_info(typeStruct->typinput, &(prodesc->result_in_func)); + fmgr_info_cxt(typeStruct->typinput, + &(prodesc->result_in_func), + proc_cxt); prodesc->result_typioparam = getTypeIOParam(typeTup); ReleaseSysCache(typeTup); @@ -1422,37 +1440,26 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, /************************************************************ * Get the required information for output conversion - * of all procedure arguments + * of all procedure arguments, and set up argument naming info. ************************************************************/ if (!is_trigger && !is_event_trigger) { - prodesc->nargs = procStruct->pronargs; proc_internal_args[0] = '\0'; for (i = 0; i < prodesc->nargs; i++) { typeTup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(procStruct->proargtypes.values[i])); if (!HeapTupleIsValid(typeTup)) - { - free(prodesc->user_proname); - free(prodesc->internal_proname); - free(prodesc); elog(ERROR, "cache lookup failed for type %u", procStruct->proargtypes.values[i]); - } typeStruct = (Form_pg_type) GETSTRUCT(typeTup); /* Disallow pseudotype argument */ if (typeStruct->typtype == TYPTYPE_PSEUDO) - { - free(prodesc->user_proname); - free(prodesc->internal_proname); - free(prodesc); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("PL/Tcl functions cannot accept type %s", format_type_be(procStruct->proargtypes.values[i])))); - } if (typeStruct->typtype == TYPTYPE_COMPOSITE) { @@ -1462,8 +1469,9 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, else { prodesc->arg_is_rowtype[i] = false; - perm_fmgr_info(typeStruct->typoutput, - &(prodesc->arg_out_func[i])); + fmgr_info_cxt(typeStruct->typoutput, + &(prodesc->arg_out_func[i]), + proc_cxt); snprintf(buf, sizeof(buf), "%d", i + 1); } @@ -1490,12 +1498,10 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, * Create the tcl command to define the internal * procedure * - * leave this code as DString - it's a text processing function - * that only gets invoked when the tcl function is invoked - * for the first time + * Leave this code as DString - performance is not critical here, + * and we don't want to duplicate the knowledge of the Tcl quoting + * rules that's embedded in Tcl_DStringAppendElement. ************************************************************/ - Tcl_DStringInit(&proc_internal_def); - Tcl_DStringInit(&proc_internal_body); Tcl_DStringAppendElement(&proc_internal_def, "proc"); Tcl_DStringAppendElement(&proc_internal_def, internal_proname); Tcl_DStringAppendElement(&proc_internal_def, proc_internal_args); @@ -1514,7 +1520,6 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, "array set NEW $__PLTcl_Tup_NEW\n", -1); Tcl_DStringAppend(&proc_internal_body, "array set OLD $__PLTcl_Tup_OLD\n", -1); - Tcl_DStringAppend(&proc_internal_body, "set i 0\n" "set v 0\n" @@ -1556,7 +1561,6 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, pfree(proc_source); Tcl_DStringAppendElement(&proc_internal_def, Tcl_DStringValue(&proc_internal_body)); - Tcl_DStringFree(&proc_internal_body); /************************************************************ * Create the procedure in the interpreter @@ -1565,28 +1569,52 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, Tcl_DStringValue(&proc_internal_def), Tcl_DStringLength(&proc_internal_def), TCL_EVAL_GLOBAL); - Tcl_DStringFree(&proc_internal_def); if (tcl_rc != TCL_OK) - { - free(prodesc->user_proname); - free(prodesc->internal_proname); - free(prodesc); ereport(ERROR, (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), errmsg("could not create internal procedure \"%s\": %s", internal_proname, utf_u2e(Tcl_GetStringResult(interp))))); - } + } + PG_CATCH(); + { + /* + * If we failed anywhere above, clean up whatever got allocated. It + * should all be in the proc_cxt, except for the DStrings. + */ + if (proc_cxt) + MemoryContextDelete(proc_cxt); + Tcl_DStringFree(&proc_internal_def); + Tcl_DStringFree(&proc_internal_body); + PG_RE_THROW(); + } + PG_END_TRY(); - /************************************************************ - * Add the proc description block to the hashtable. Note we do not - * attempt to free any previously existing prodesc block. This is - * annoying, but necessary since there could be active calls using - * the old prodesc. - ************************************************************/ - proc_ptr->proc_ptr = prodesc; + /* + * Install the new proc description block in the hashtable, incrementing + * its refcount (the hashtable link counts as a reference). Then, if + * there was a previous definition of the function, decrement that one's + * refcount, and delete it if no longer referenced. The order of + * operations here is important: if something goes wrong during the + * MemoryContextDelete, leaking some memory for the old definition is OK, + * but we don't want to corrupt the live hashtable entry. (Likewise, + * freeing the DStrings is pretty low priority if that happens.) + */ + old_prodesc = proc_ptr->proc_ptr; + + proc_ptr->proc_ptr = prodesc; + prodesc->fn_refcount++; + + if (old_prodesc != NULL) + { + Assert(old_prodesc->fn_refcount > 0); + if (--old_prodesc->fn_refcount == 0) + MemoryContextDelete(old_prodesc->fn_cxt); } + Tcl_DStringFree(&proc_internal_def); + Tcl_DStringFree(&proc_internal_body); + ReleaseSysCache(procTup); return prodesc; From 6f7c0ea32f808a7dad3ec07db7e5fdf6514d2af0 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 31 Aug 2016 19:54:58 -0400 Subject: [PATCH 095/871] Improve memory management for PL/Perl functions. Unlike PL/Tcl, PL/Perl at least made an attempt to clean up after itself when a function gets redefined. But it was still using TopMemoryContext for the fn_mcxt of argument/result I/O functions, resulting in the potential for memory leaks depending on what those functions did, and the retail alloc/free logic was pretty bulky as well. Fix things to use a per-function memory context like the other PLs now do. Tweak a couple of places where things were being done in a not-very-safe order (on the principle that a memory leak is better than leaving global state inconsistent after an error). Also make some minor cosmetic adjustments, mostly in field names, to make the code look similar to the way PL/Tcl does now wherever it's essentially the same logic. Michael Paquier and Tom Lane Discussion: --- src/pl/plperl/plperl.c | 237 +++++++++++++++++++++-------------------- 1 file changed, 122 insertions(+), 115 deletions(-) diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 2cd761496d..87113f0fb1 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -98,17 +98,19 @@ typedef struct plperl_interp_desc /********************************************************************** * The information we cache about loaded procedures * - * The refcount field counts the struct's reference from the hash table shown - * below, plus one reference for each function call level that is using the - * struct. We can release the struct, and the associated Perl sub, when the - * refcount goes to zero. + * The fn_refcount field counts the struct's reference from the hash table + * shown below, plus one reference for each function call level that is using + * the struct. We can release the struct, and the associated Perl sub, when + * the fn_refcount goes to zero. Releasing the struct itself is done by + * deleting the fn_cxt, which also gets rid of all subsidiary data. **********************************************************************/ typedef struct plperl_proc_desc { char *proname; /* user name of procedure */ + MemoryContext fn_cxt; /* memory context for this procedure */ + unsigned long fn_refcount; /* number of active references */ TransactionId fn_xmin; /* xmin/TID of procedure's pg_proc tuple */ ItemPointerData fn_tid; - int refcount; /* reference count of this struct */ SV *reference; /* CODE reference for Perl sub */ plperl_interp_desc *interp; /* interpreter it's created in */ bool fn_readonly; /* is function readonly (not volatile)? */ @@ -122,18 +124,19 @@ typedef struct plperl_proc_desc Oid result_oid; /* Oid of result type */ FmgrInfo result_in_func; /* I/O function and arg for result type */ Oid result_typioparam; - /* Conversion info for function's argument types: */ + /* Per-argument info for function's argument types: */ int nargs; - FmgrInfo arg_out_func[FUNC_MAX_ARGS]; - bool arg_is_rowtype[FUNC_MAX_ARGS]; - Oid arg_arraytype[FUNC_MAX_ARGS]; /* InvalidOid if not an array */ + FmgrInfo *arg_out_func; /* output fns for arg types */ + bool *arg_is_rowtype; /* is each arg composite? */ + Oid *arg_arraytype; /* InvalidOid if not an array */ } plperl_proc_desc; #define increment_prodesc_refcount(prodesc) \ - ((prodesc)->refcount++) + ((prodesc)->fn_refcount++) #define decrement_prodesc_refcount(prodesc) \ do { \ - if (--((prodesc)->refcount) <= 0) \ + Assert((prodesc)->fn_refcount > 0); \ + if (--((prodesc)->fn_refcount) == 0) \ free_plperl_function(prodesc); \ } while(0) @@ -353,23 +356,6 @@ hek2cstr(HE *he) return ret; } -/* - * This routine is a crock, and so is everyplace that calls it. The problem - * is that the cached form of plperl functions/queries is allocated permanently - * (mostly via malloc()) and never released until backend exit. Subsidiary - * data structures such as fmgr info records therefore must live forever - * as well. A better implementation would store all this stuff in a per- - * function memory context that could be reclaimed at need. In the meantime, - * fmgr_info_cxt must be called specifying TopMemoryContext so that whatever - * it might allocate, and whatever the eventual function might allocate using - * fn_mcxt, will live forever too. - */ -static void -perm_fmgr_info(Oid functionId, FmgrInfo *finfo) -{ - fmgr_info_cxt(functionId, finfo, TopMemoryContext); -} - /* * _PG_init() - library load-time initialization @@ -1433,6 +1419,10 @@ plperl_ref_from_pg_array(Datum arg, Oid typid) SV *av; HV *hv; + /* + * Currently we make no effort to cache any of the stuff we look up here, + * which is bad. + */ info = palloc0(sizeof(plperl_array_info)); /* get element type information, including output conversion function */ @@ -1440,10 +1430,16 @@ plperl_ref_from_pg_array(Datum arg, Oid typid) &typlen, &typbyval, &typalign, &typdelim, &typioparam, &typoutputfunc); - if ((transform_funcid = get_transform_fromsql(elementtype, current_call_data->prodesc->lang_oid, current_call_data->prodesc->trftypes))) - perm_fmgr_info(transform_funcid, &info->transform_proc); + /* Check for a transform function */ + transform_funcid = get_transform_fromsql(elementtype, + current_call_data->prodesc->lang_oid, + current_call_data->prodesc->trftypes); + + /* Look up transform or output function as appropriate */ + if (OidIsValid(transform_funcid)) + fmgr_info(transform_funcid, &info->transform_proc); else - perm_fmgr_info(typoutputfunc, &info->proc); + fmgr_info(typoutputfunc, &info->proc); info->elem_is_rowtype = type_is_rowtype(elementtype); @@ -1791,18 +1787,18 @@ plperl_call_handler(PG_FUNCTION_ARGS) } PG_CATCH(); { - if (this_call_data.prodesc) - decrement_prodesc_refcount(this_call_data.prodesc); current_call_data = save_call_data; activate_interpreter(oldinterp); + if (this_call_data.prodesc) + decrement_prodesc_refcount(this_call_data.prodesc); PG_RE_THROW(); } PG_END_TRY(); - if (this_call_data.prodesc) - decrement_prodesc_refcount(this_call_data.prodesc); current_call_data = save_call_data; activate_interpreter(oldinterp); + if (this_call_data.prodesc) + decrement_prodesc_refcount(this_call_data.prodesc); return retval; } @@ -2616,7 +2612,7 @@ validate_plperl_function(plperl_proc_ptr *proc_ptr, HeapTuple procTup) static void free_plperl_function(plperl_proc_desc *prodesc) { - Assert(prodesc->refcount <= 0); + Assert(prodesc->fn_refcount == 0); /* Release CODE reference, if we have one, from the appropriate interp */ if (prodesc->reference) { @@ -2626,12 +2622,8 @@ free_plperl_function(plperl_proc_desc *prodesc) SvREFCNT_dec(prodesc->reference); activate_interpreter(oldinterp); } - /* Get rid of what we conveniently can of our own structs */ - /* (FmgrInfo subsidiary info will get leaked ...) */ - if (prodesc->proname) - free(prodesc->proname); - list_free(prodesc->trftypes); - free(prodesc); + /* Release all PG-owned data for this proc */ + MemoryContextDelete(prodesc->fn_cxt); } @@ -2642,8 +2634,8 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) Form_pg_proc procStruct; plperl_proc_key proc_key; plperl_proc_ptr *proc_ptr; - plperl_proc_desc *prodesc = NULL; - int i; + plperl_proc_desc *volatile prodesc = NULL; + volatile MemoryContext proc_cxt = NULL; plperl_interp_desc *oldinterp = plperl_active_interp; ErrorContextCallback plperl_error_context; @@ -2653,41 +2645,50 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) elog(ERROR, "cache lookup failed for function %u", fn_oid); procStruct = (Form_pg_proc) GETSTRUCT(procTup); - /* Set a callback for reporting compilation errors */ - plperl_error_context.callback = plperl_compile_callback; - plperl_error_context.previous = error_context_stack; - plperl_error_context.arg = NameStr(procStruct->proname); - error_context_stack = &plperl_error_context; - - /* Try to find function in plperl_proc_hash */ + /* + * Try to find function in plperl_proc_hash. The reason for this + * overcomplicated-seeming lookup procedure is that we don't know whether + * it's plperl or plperlu, and don't want to spend a lookup in pg_language + * to find out. + */ proc_key.proc_id = fn_oid; proc_key.is_trigger = is_trigger; proc_key.user_id = GetUserId(); - proc_ptr = hash_search(plperl_proc_hash, &proc_key, HASH_FIND, NULL); + if (validate_plperl_function(proc_ptr, procTup)) + { + /* Found valid plperl entry */ + ReleaseSysCache(procTup); + return proc_ptr->proc_ptr; + } + /* If not found or obsolete, maybe it's plperlu */ + proc_key.user_id = InvalidOid; + proc_ptr = hash_search(plperl_proc_hash, &proc_key, + HASH_FIND, NULL); if (validate_plperl_function(proc_ptr, procTup)) - prodesc = proc_ptr->proc_ptr; - else { - /* If not found or obsolete, maybe it's plperlu */ - proc_key.user_id = InvalidOid; - proc_ptr = hash_search(plperl_proc_hash, &proc_key, - HASH_FIND, NULL); - if (validate_plperl_function(proc_ptr, procTup)) - prodesc = proc_ptr->proc_ptr; + /* Found valid plperlu entry */ + ReleaseSysCache(procTup); + return proc_ptr->proc_ptr; } /************************************************************ * If we haven't found it in the hashtable, we analyze * the function's arguments and return type and store - * the in-/out-functions in the prodesc block and create - * a new hashtable entry for it. - * - * Then we load the procedure into the Perl interpreter. + * the in-/out-functions in the prodesc block, + * then we load the procedure into the Perl interpreter, + * and last we create a new hashtable entry for it. ************************************************************/ - if (prodesc == NULL) + + /* Set a callback for reporting compilation errors */ + plperl_error_context.callback = plperl_compile_callback; + plperl_error_context.previous = error_context_stack; + plperl_error_context.arg = NameStr(procStruct->proname); + error_context_stack = &plperl_error_context; + + PG_TRY(); { HeapTuple langTup; HeapTuple typeTup; @@ -2697,42 +2698,42 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) Datum prosrcdatum; bool isnull; char *proc_source; + MemoryContext oldcontext; /************************************************************ - * Allocate a new procedure description block + * Allocate a context that will hold all PG data for the procedure. ************************************************************/ - prodesc = (plperl_proc_desc *) malloc(sizeof(plperl_proc_desc)); - if (prodesc == NULL) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"))); - /* Initialize all fields to 0 so free_plperl_function is safe */ - MemSet(prodesc, 0, sizeof(plperl_proc_desc)); + proc_cxt = AllocSetContextCreate(TopMemoryContext, + NameStr(procStruct->proname), + ALLOCSET_SMALL_SIZES); - prodesc->proname = strdup(NameStr(procStruct->proname)); - if (prodesc->proname == NULL) - { - free_plperl_function(prodesc); - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"))); - } + /************************************************************ + * Allocate and fill a new procedure description block. + * struct prodesc and subsidiary data must all live in proc_cxt. + ************************************************************/ + oldcontext = MemoryContextSwitchTo(proc_cxt); + prodesc = (plperl_proc_desc *) palloc0(sizeof(plperl_proc_desc)); + prodesc->proname = pstrdup(NameStr(procStruct->proname)); + prodesc->fn_cxt = proc_cxt; + prodesc->fn_refcount = 0; prodesc->fn_xmin = HeapTupleHeaderGetRawXmin(procTup->t_data); prodesc->fn_tid = procTup->t_self; + prodesc->nargs = procStruct->pronargs; + prodesc->arg_out_func = (FmgrInfo *) palloc0(prodesc->nargs * sizeof(FmgrInfo)); + prodesc->arg_is_rowtype = (bool *) palloc0(prodesc->nargs * sizeof(bool)); + prodesc->arg_arraytype = (Oid *) palloc0(prodesc->nargs * sizeof(Oid)); + MemoryContextSwitchTo(oldcontext); /* Remember if function is STABLE/IMMUTABLE */ prodesc->fn_readonly = (procStruct->provolatile != PROVOLATILE_VOLATILE); - { - MemoryContext oldcxt; - - protrftypes_datum = SysCacheGetAttr(PROCOID, procTup, + /* Fetch protrftypes */ + protrftypes_datum = SysCacheGetAttr(PROCOID, procTup, Anum_pg_proc_protrftypes, &isnull); - oldcxt = MemoryContextSwitchTo(TopMemoryContext); - prodesc->trftypes = isnull ? NIL : oid_array_to_list(protrftypes_datum); - MemoryContextSwitchTo(oldcxt); - } + MemoryContextSwitchTo(proc_cxt); + prodesc->trftypes = isnull ? NIL : oid_array_to_list(protrftypes_datum); + MemoryContextSwitchTo(oldcontext); /************************************************************ * Lookup the pg_language tuple by Oid @@ -2740,11 +2741,8 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) langTup = SearchSysCache1(LANGOID, ObjectIdGetDatum(procStruct->prolang)); if (!HeapTupleIsValid(langTup)) - { - free_plperl_function(prodesc); elog(ERROR, "cache lookup failed for language %u", procStruct->prolang); - } langStruct = (Form_pg_language) GETSTRUCT(langTup); prodesc->lang_oid = HeapTupleGetOid(langTup); prodesc->lanpltrusted = langStruct->lanpltrusted; @@ -2760,11 +2758,8 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) SearchSysCache1(TYPEOID, ObjectIdGetDatum(procStruct->prorettype)); if (!HeapTupleIsValid(typeTup)) - { - free_plperl_function(prodesc); elog(ERROR, "cache lookup failed for type %u", procStruct->prorettype); - } typeStruct = (Form_pg_type) GETSTRUCT(typeTup); /* Disallow pseudotype result, except VOID or RECORD */ @@ -2775,21 +2770,15 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) /* okay */ ; else if (procStruct->prorettype == TRIGGEROID || procStruct->prorettype == EVTTRIGGEROID) - { - free_plperl_function(prodesc); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("trigger functions can only be called " "as triggers"))); - } else - { - free_plperl_function(prodesc); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("PL/Perl functions cannot return type %s", format_type_be(procStruct->prorettype)))); - } } prodesc->result_oid = procStruct->prorettype; @@ -2800,7 +2789,9 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) prodesc->fn_retisarray = (typeStruct->typlen == -1 && typeStruct->typelem); - perm_fmgr_info(typeStruct->typinput, &(prodesc->result_in_func)); + fmgr_info_cxt(typeStruct->typinput, + &(prodesc->result_in_func), + proc_cxt); prodesc->result_typioparam = getTypeIOParam(typeTup); ReleaseSysCache(typeTup); @@ -2812,29 +2803,24 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) ************************************************************/ if (!is_trigger && !is_event_trigger) { - prodesc->nargs = procStruct->pronargs; + int i; + for (i = 0; i < prodesc->nargs; i++) { typeTup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(procStruct->proargtypes.values[i])); if (!HeapTupleIsValid(typeTup)) - { - free_plperl_function(prodesc); elog(ERROR, "cache lookup failed for type %u", procStruct->proargtypes.values[i]); - } typeStruct = (Form_pg_type) GETSTRUCT(typeTup); /* Disallow pseudotype argument */ if (typeStruct->typtype == TYPTYPE_PSEUDO && procStruct->proargtypes.values[i] != RECORDOID) - { - free_plperl_function(prodesc); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("PL/Perl functions cannot accept type %s", format_type_be(procStruct->proargtypes.values[i])))); - } if (typeStruct->typtype == TYPTYPE_COMPOSITE || procStruct->proargtypes.values[i] == RECORDOID) @@ -2842,8 +2828,9 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) else { prodesc->arg_is_rowtype[i] = false; - perm_fmgr_info(typeStruct->typoutput, - &(prodesc->arg_out_func[i])); + fmgr_info_cxt(typeStruct->typoutput, + &(prodesc->arg_out_func[i]), + proc_cxt); } /* Identify array attributes */ @@ -2880,22 +2867,42 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) activate_interpreter(oldinterp); pfree(proc_source); + if (!prodesc->reference) /* can this happen? */ - { - free_plperl_function(prodesc); elog(ERROR, "could not create PL/Perl internal procedure"); - } /************************************************************ - * OK, link the procedure into the correct hashtable entry + * OK, link the procedure into the correct hashtable entry. + * Note we assume that the hashtable entry either doesn't exist yet, + * or we already cleared its proc_ptr during the validation attempts + * above. So no need to decrement an old refcount here. ************************************************************/ proc_key.user_id = prodesc->lanpltrusted ? GetUserId() : InvalidOid; proc_ptr = hash_search(plperl_proc_hash, &proc_key, HASH_ENTER, NULL); + /* We assume these two steps can't throw an error: */ proc_ptr->proc_ptr = prodesc; increment_prodesc_refcount(prodesc); } + PG_CATCH(); + { + /* + * If we got as far as creating a reference, we should be able to use + * free_plperl_function() to clean up. If not, then at most we have + * some PG memory resources in proc_cxt, which we can just delete. + */ + if (prodesc && prodesc->reference) + free_plperl_function(prodesc); + else if (proc_cxt) + MemoryContextDelete(proc_cxt); + + /* Be sure to restore the previous interpreter, too, for luck */ + activate_interpreter(oldinterp); + + PG_RE_THROW(); + } + PG_END_TRY(); /* restore previous error callback */ error_context_stack = plperl_error_context.previous; From 6c03d981a6b64ed8caaed4e94b54ef926202c9f3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 1 Sep 2016 10:13:55 -0400 Subject: [PATCH 096/871] Change API of ShmemAlloc() so it throws error rather than returning NULL. A majority of callers seem to have believed that this was the API spec already, because they omitted any check for a NULL result, and hence would crash on an out-of-shared-memory failure. The original proposal was to just add such error checks everywhere, but that does nothing to prevent similar omissions in future. Instead, let's make ShmemAlloc() throw the error (so we can remove the caller-side checks that do exist), and introduce a new function ShmemAllocNoError() that has the previous behavior of returning NULL, for the small number of callers that need that and are prepared to do the right thing. This also lets us remove the rather wishy-washy behavior of printing a WARNING for out-of-shmem, which never made much sense: either the caller has a strategy for dealing with that, or it doesn't. It's not ShmemAlloc's business to decide whether a warning is appropriate. The v10 release notes will need to call this out as a significant source-code change. It's likely that it will be a bug fix for extension callers too, but if not, they'll need to change to using ShmemAllocNoError(). This is nominally a bug fix, but the odds that it's fixing any live bug are actually rather small, because in general the requests being made by the unchecked callers were already accounted for in determining the overall shmem size, so really they ought not fail. Between that and the possible impact on extensions, no back-patch. Discussion: <24843.1472563085@sss.pgh.pa.us> --- src/backend/storage/ipc/shmem.c | 41 ++++++++++++++++------------ src/backend/storage/lmgr/predicate.c | 12 -------- src/backend/storage/lmgr/proc.c | 6 +--- src/include/storage/shmem.h | 1 + 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/src/backend/storage/ipc/shmem.c b/src/backend/storage/ipc/shmem.c index 1efe0201a7..cc3af2d615 100644 --- a/src/backend/storage/ipc/shmem.c +++ b/src/backend/storage/ipc/shmem.c @@ -163,14 +163,31 @@ InitShmemAllocation(void) /* * ShmemAlloc -- allocate max-aligned chunk from shared memory * - * Assumes ShmemLock and ShmemSegHdr are initialized. + * Throws error if request cannot be satisfied. * - * Returns: real pointer to memory or NULL if we are out - * of space. Has to return a real pointer in order - * to be compatible with malloc(). + * Assumes ShmemLock and ShmemSegHdr are initialized. */ void * ShmemAlloc(Size size) +{ + void *newSpace; + + newSpace = ShmemAllocNoError(size); + if (!newSpace) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of shared memory (%zu bytes requested)", + size))); + return newSpace; +} + +/* + * ShmemAllocNoError -- allocate max-aligned chunk from shared memory + * + * As ShmemAlloc, but returns NULL if out of space, rather than erroring. + */ +void * +ShmemAllocNoError(Size size) { Size newStart; Size newFree; @@ -206,11 +223,7 @@ ShmemAlloc(Size size) SpinLockRelease(ShmemLock); - if (!newSpace) - ereport(WARNING, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of shared memory"))); - + /* note this assert is okay with newSpace == NULL */ Assert(newSpace == (void *) CACHELINEALIGN(newSpace)); return newSpace; @@ -293,7 +306,7 @@ ShmemInitHash(const char *name, /* table string name for shmem index */ * The shared memory allocator must be specified too. */ infoP->dsize = infoP->max_dsize = hash_select_dirsize(max_size); - infoP->alloc = ShmemAlloc; + infoP->alloc = ShmemAllocNoError; hash_flags |= HASH_SHARED_MEM | HASH_ALLOC | HASH_DIRSIZE; /* look it up in the shmem index */ @@ -364,12 +377,6 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr) */ Assert(shmemseghdr->index == NULL); structPtr = ShmemAlloc(size); - if (structPtr == NULL) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("not enough shared memory for data structure" - " \"%s\" (%zu bytes requested)", - name, size))); shmemseghdr->index = structPtr; *foundPtr = FALSE; } @@ -410,7 +417,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr) else { /* It isn't in the table yet. allocate and initialize it */ - structPtr = ShmemAlloc(size); + structPtr = ShmemAllocNoError(size); if (structPtr == NULL) { /* out of memory; remove the failed ShmemIndex entry */ diff --git a/src/backend/storage/lmgr/predicate.c b/src/backend/storage/lmgr/predicate.c index 7cdb35541b..4064b2033c 100644 --- a/src/backend/storage/lmgr/predicate.c +++ b/src/backend/storage/lmgr/predicate.c @@ -1184,12 +1184,6 @@ InitPredicateLocks(void) requestSize = mul_size((Size) max_table_size, PredXactListElementDataSize); PredXact->element = ShmemAlloc(requestSize); - if (PredXact->element == NULL) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("not enough shared memory for elements of data structure" - " \"%s\" (%zu bytes requested)", - "PredXactList", requestSize))); /* Add all elements to available list, clean. */ memset(PredXact->element, 0, requestSize); for (i = 0; i < max_table_size; i++) @@ -1255,12 +1249,6 @@ InitPredicateLocks(void) requestSize = mul_size((Size) max_table_size, RWConflictDataSize); RWConflictPool->element = ShmemAlloc(requestSize); - if (RWConflictPool->element == NULL) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("not enough shared memory for elements of data structure" - " \"%s\" (%zu bytes requested)", - "RWConflictPool", requestSize))); /* Add all elements to available list, clean. */ memset(RWConflictPool->element, 0, requestSize); for (i = 0; i < max_table_size; i++) diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 9a758bd916..33e7023656 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -194,14 +194,10 @@ InitProcGlobal(void) * between groups. */ procs = (PGPROC *) ShmemAlloc(TotalProcs * sizeof(PGPROC)); + MemSet(procs, 0, TotalProcs * sizeof(PGPROC)); ProcGlobal->allProcs = procs; /* XXX allProcCount isn't really all of them; it excludes prepared xacts */ ProcGlobal->allProcCount = MaxBackends + NUM_AUXILIARY_PROCS; - if (!procs) - ereport(FATAL, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of shared memory"))); - MemSet(procs, 0, TotalProcs * sizeof(PGPROC)); /* * Also allocate a separate array of PGXACT structures. This is separate diff --git a/src/include/storage/shmem.h b/src/include/storage/shmem.h index 6468e6627a..2560e6c6da 100644 --- a/src/include/storage/shmem.h +++ b/src/include/storage/shmem.h @@ -35,6 +35,7 @@ typedef struct SHM_QUEUE extern void InitShmemAccess(void *seghdr); extern void InitShmemAllocation(void); extern void *ShmemAlloc(Size size); +extern void *ShmemAllocNoError(Size size); extern bool ShmemAddrIsValid(const void *addr); extern void InitShmemIndex(void); extern HTAB *ShmemInitHash(const char *name, long init_size, long max_size, From 76f9dd4fa82270899f7b56b002b5d34226dc99d8 Mon Sep 17 00:00:00 2001 From: Kevin Grittner Date: Thu, 1 Sep 2016 16:10:30 -0500 Subject: [PATCH 097/871] Improve tab completion for BEGIN & START|SET TRANSACTION. Andreas Karlsson with minor change by me for SET TRANSACTION SNAPSHOT. --- src/bin/psql/tab-complete.c | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 1345e4ed80..019f75a376 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1894,8 +1894,11 @@ psql_completion(const char *text, int start, int end) else if (Matches5("ALTER", "GROUP", MatchAny, "ADD|DROP", "USER")) COMPLETE_WITH_QUERY(Query_for_list_of_roles); -/* BEGIN, END, ABORT */ - else if (Matches1("BEGIN|END|ABORT")) +/* BEGIN */ + else if (Matches1("BEGIN")) + COMPLETE_WITH_LIST6("WORK", "TRANSACTION", "ISOLATION LEVEL", "READ", "DEFERRABLE", "NOT DEFERRABLE"); +/* END, ABORT */ + else if (Matches1("END|ABORT")) COMPLETE_WITH_LIST2("WORK", "TRANSACTION"); /* COMMIT */ else if (Matches1("COMMIT")) @@ -2762,20 +2765,36 @@ psql_completion(const char *text, int start, int end) else if (Matches1("SHOW")) COMPLETE_WITH_QUERY(Query_for_list_of_show_vars); /* Complete "SET TRANSACTION" */ - else if (Matches2("SET|BEGIN|START", "TRANSACTION") || + else if (Matches2("SET", "TRANSACTION")) + COMPLETE_WITH_LIST5("SNAPSHOT", "ISOLATION LEVEL", "READ", "DEFERRABLE", "NOT DEFERRABLE"); + else if (Matches2("BEGIN|START", "TRANSACTION") || Matches2("BEGIN", "WORK") || + Matches1("BEGIN") || Matches5("SET", "SESSION", "CHARACTERISTICS", "AS", "TRANSACTION")) - COMPLETE_WITH_LIST2("ISOLATION LEVEL", "READ"); + COMPLETE_WITH_LIST4("ISOLATION LEVEL", "READ", "DEFERRABLE", "NOT DEFERRABLE"); + else if (Matches3("SET|BEGIN|START", "TRANSACTION|WORK", "NOT") || + Matches2("BEGIN", "NOT") || + Matches6("SET", "SESSION", "CHARACTERISTICS", "AS", "TRANSACTION", "NOT")) + COMPLETE_WITH_CONST("DEFERRABLE"); else if (Matches3("SET|BEGIN|START", "TRANSACTION|WORK", "ISOLATION") || + Matches2("BEGIN", "ISOLATION") || Matches6("SET", "SESSION", "CHARACTERISTICS", "AS", "TRANSACTION", "ISOLATION")) COMPLETE_WITH_CONST("LEVEL"); - else if (Matches4("SET|BEGIN|START", "TRANSACTION|WORK", "ISOLATION", "LEVEL")) + else if (Matches4("SET|BEGIN|START", "TRANSACTION|WORK", "ISOLATION", "LEVEL") || + Matches3("BEGIN", "ISOLATION", "LEVEL") || + Matches7("SET", "SESSION", "CHARACTERISTICS", "AS", "TRANSACTION", "ISOLATION", "LEVEL")) COMPLETE_WITH_LIST3("READ", "REPEATABLE READ", "SERIALIZABLE"); - else if (Matches5("SET|BEGIN|START", "TRANSACTION|WORK", "ISOLATION", "LEVEL", "READ")) + else if (Matches5("SET|BEGIN|START", "TRANSACTION|WORK", "ISOLATION", "LEVEL", "READ") || + Matches4("BEGIN", "ISOLATION", "LEVEL", "READ") || + Matches8("SET", "SESSION", "CHARACTERISTICS", "AS", "TRANSACTION", "ISOLATION", "LEVEL", "READ")) COMPLETE_WITH_LIST2("UNCOMMITTED", "COMMITTED"); - else if (Matches5("SET|BEGIN|START", "TRANSACTION|WORK", "ISOLATION", "LEVEL", "REPEATABLE")) + else if (Matches5("SET|BEGIN|START", "TRANSACTION|WORK", "ISOLATION", "LEVEL", "REPEATABLE") || + Matches4("BEGIN", "ISOLATION", "LEVEL", "REPEATABLE") || + Matches8("SET", "SESSION", "CHARACTERISTICS", "AS", "TRANSACTION", "ISOLATION", "LEVEL", "REPEATABLE")) COMPLETE_WITH_CONST("READ"); - else if (Matches3("SET|BEGIN|START", "TRANSACTION|WORK", "READ")) + else if (Matches3("SET|BEGIN|START", "TRANSACTION|WORK", "READ") || + Matches2("BEGIN", "READ") || + Matches6("SET", "SESSION", "CHARACTERISTICS", "AS", "TRANSACTION", "READ")) COMPLETE_WITH_LIST2("ONLY", "WRITE"); /* SET CONSTRAINTS */ else if (Matches2("SET", "CONSTRAINTS")) From 9f85784cae4d057f307b83b0d33edede33434f04 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 2 Sep 2016 08:39:39 +0300 Subject: [PATCH 098/871] Support multiple iterators in the Red-Black Tree implementation. While we don't need multiple iterators at the moment, the interface is nicer and less dangerous this way. Aleksander Alekseev, with some changes by me. --- src/backend/access/gin/ginbulk.c | 4 +- src/backend/lib/rbtree.c | 333 ++++++++++++++++--------------- src/include/access/gin_private.h | 1 + src/include/lib/rbtree.h | 22 +- 4 files changed, 197 insertions(+), 163 deletions(-) diff --git a/src/backend/access/gin/ginbulk.c b/src/backend/access/gin/ginbulk.c index d6422ea91e..71c64e468c 100644 --- a/src/backend/access/gin/ginbulk.c +++ b/src/backend/access/gin/ginbulk.c @@ -255,7 +255,7 @@ qsortCompareItemPointers(const void *a, const void *b) void ginBeginBAScan(BuildAccumulator *accum) { - rb_begin_iterate(accum->tree, LeftRightWalk); + rb_begin_iterate(accum->tree, LeftRightWalk, &accum->tree_walk); } /* @@ -271,7 +271,7 @@ ginGetBAEntry(BuildAccumulator *accum, GinEntryAccumulator *entry; ItemPointerData *list; - entry = (GinEntryAccumulator *) rb_iterate(accum->tree); + entry = (GinEntryAccumulator *) rb_iterate(&accum->tree_walk); if (entry == NULL) return NULL; /* no more entries */ diff --git a/src/backend/lib/rbtree.c b/src/backend/lib/rbtree.c index 4fa8a1dd58..242e9803cc 100644 --- a/src/backend/lib/rbtree.c +++ b/src/backend/lib/rbtree.c @@ -29,17 +29,6 @@ #include "lib/rbtree.h" -/* - * Values of RBNode.iteratorState - * - * Note that iteratorState has an undefined value except in nodes that are - * currently being visited by an active iteration. - */ -#define InitialState (0) -#define FirstStepDone (1) -#define SecondStepDone (2) -#define ThirdStepDone (3) - /* * Colors of nodes (values of RBNode.color) */ @@ -53,10 +42,6 @@ struct RBTree { RBNode *root; /* root node, or RBNIL if tree is empty */ - /* Iteration state */ - RBNode *cur; /* current iteration node */ - RBNode *(*iterate) (RBTree *rb); - /* Remaining fields are constant after rb_create */ Size node_size; /* actual size of tree nodes */ @@ -75,8 +60,19 @@ struct RBTree */ #define RBNIL (&sentinel) -static RBNode sentinel = {InitialState, RBBLACK, RBNIL, RBNIL, NULL}; +static RBNode sentinel = {RBBLACK, RBNIL, RBNIL, NULL}; +/* + * Values used in the RBTreeIterator.next_state field, with an + * InvertedWalk iterator. + */ +typedef enum InvertedWalkNextStep +{ + NextStepBegin, + NextStepUp, + NextStepLeft, + NextStepRight +} InvertedWalkNextStep; /* * rb_create: create an empty RBTree @@ -123,8 +119,6 @@ rb_create(Size node_size, Assert(node_size > sizeof(RBNode)); tree->root = RBNIL; - tree->cur = RBNIL; - tree->iterate = NULL; tree->node_size = node_size; tree->comparator = comparator; tree->combiner = combiner; @@ -437,7 +431,6 @@ rb_insert(RBTree *rb, const RBNode *data, bool *isNew) x = rb->allocfunc (rb->arg); - x->iteratorState = InitialState; x->color = RBRED; x->left = RBNIL; @@ -653,171 +646,194 @@ rb_delete(RBTree *rb, RBNode *node) * Traverse * **********************************************************************/ -/* - * The iterator routines were originally coded in tail-recursion style, - * which is nice to look at, but is trouble if your compiler isn't smart - * enough to optimize it. Now we just use looping. - */ -#define descend(next_node) \ - do { \ - (next_node)->iteratorState = InitialState; \ - node = rb->cur = (next_node); \ - goto restart; \ - } while (0) +static RBNode * +rb_left_right_iterator(RBTreeIterator *iter) +{ + if (iter->last_visited == NULL) + { + iter->last_visited = iter->rb->root; + while (iter->last_visited->left != RBNIL) + iter->last_visited = iter->last_visited->left; -#define ascend(next_node) \ - do { \ - node = rb->cur = (next_node); \ - goto restart; \ - } while (0) + return iter->last_visited; + } + if (iter->last_visited->right != RBNIL) + { + iter->last_visited = iter->last_visited->right; + while (iter->last_visited->left != RBNIL) + iter->last_visited = iter->last_visited->left; -static RBNode * -rb_left_right_iterator(RBTree *rb) -{ - RBNode *node = rb->cur; + return iter->last_visited; + } -restart: - switch (node->iteratorState) + for (;;) { - case InitialState: - if (node->left != RBNIL) - { - node->iteratorState = FirstStepDone; - descend(node->left); - } - /* FALL THROUGH */ - case FirstStepDone: - node->iteratorState = SecondStepDone; - return node; - case SecondStepDone: - if (node->right != RBNIL) - { - node->iteratorState = ThirdStepDone; - descend(node->right); - } - /* FALL THROUGH */ - case ThirdStepDone: - if (node->parent) - ascend(node->parent); + RBNode *came_from = iter->last_visited; + + iter->last_visited = iter->last_visited->parent; + if (iter->last_visited == NULL) + { + iter->is_over = true; break; - default: - elog(ERROR, "unrecognized rbtree node state: %d", - node->iteratorState); + } + + if (iter->last_visited->left == came_from) + break; /* came from left sub-tree, return current + * node */ + + /* else - came from right sub-tree, continue to move up */ } - return NULL; + return iter->last_visited; } static RBNode * -rb_right_left_iterator(RBTree *rb) +rb_right_left_iterator(RBTreeIterator *iter) { - RBNode *node = rb->cur; + if (iter->last_visited == NULL) + { + iter->last_visited = iter->rb->root; + while (iter->last_visited->right != RBNIL) + iter->last_visited = iter->last_visited->right; + + return iter->last_visited; + } -restart: - switch (node->iteratorState) + if (iter->last_visited->left != RBNIL) { - case InitialState: - if (node->right != RBNIL) - { - node->iteratorState = FirstStepDone; - descend(node->right); - } - /* FALL THROUGH */ - case FirstStepDone: - node->iteratorState = SecondStepDone; - return node; - case SecondStepDone: - if (node->left != RBNIL) - { - node->iteratorState = ThirdStepDone; - descend(node->left); - } - /* FALL THROUGH */ - case ThirdStepDone: - if (node->parent) - ascend(node->parent); + iter->last_visited = iter->last_visited->left; + while (iter->last_visited->right != RBNIL) + iter->last_visited = iter->last_visited->right; + + return iter->last_visited; + } + + for (;;) + { + RBNode *came_from = iter->last_visited; + + iter->last_visited = iter->last_visited->parent; + if (iter->last_visited == NULL) + { + iter->is_over = true; break; - default: - elog(ERROR, "unrecognized rbtree node state: %d", - node->iteratorState); + } + + if (iter->last_visited->right == came_from) + break; /* came from right sub-tree, return current + * node */ + + /* else - came from left sub-tree, continue to move up */ } - return NULL; + return iter->last_visited; } static RBNode * -rb_direct_iterator(RBTree *rb) +rb_direct_iterator(RBTreeIterator *iter) { - RBNode *node = rb->cur; + if (iter->last_visited == NULL) + { + iter->last_visited = iter->rb->root; + return iter->last_visited; + } -restart: - switch (node->iteratorState) + if (iter->last_visited->left != RBNIL) { - case InitialState: - node->iteratorState = FirstStepDone; - return node; - case FirstStepDone: - if (node->left != RBNIL) + iter->last_visited = iter->last_visited->left; + return iter->last_visited; + } + + do + { + if (iter->last_visited->right != RBNIL) + { + iter->last_visited = iter->last_visited->right; + break; + } + + /* go up and one step right */ + for (;;) + { + RBNode *came_from = iter->last_visited; + + iter->last_visited = iter->last_visited->parent; + if (iter->last_visited == NULL) { - node->iteratorState = SecondStepDone; - descend(node->left); + iter->is_over = true; + break; } - /* FALL THROUGH */ - case SecondStepDone: - if (node->right != RBNIL) + + if ((iter->last_visited->right != came_from) && (iter->last_visited->right != RBNIL)) { - node->iteratorState = ThirdStepDone; - descend(node->right); + iter->last_visited = iter->last_visited->right; + return iter->last_visited; } - /* FALL THROUGH */ - case ThirdStepDone: - if (node->parent) - ascend(node->parent); - break; - default: - elog(ERROR, "unrecognized rbtree node state: %d", - node->iteratorState); + } } + while (iter->last_visited != NULL); - return NULL; + return iter->last_visited; } static RBNode * -rb_inverted_iterator(RBTree *rb) +rb_inverted_iterator(RBTreeIterator *iter) { - RBNode *node = rb->cur; + RBNode *came_from; -restart: - switch (node->iteratorState) +loop: + switch ((InvertedWalkNextStep) iter->next_step) { - case InitialState: - if (node->left != RBNIL) + /* First call, begin from root */ + case NextStepBegin: + iter->last_visited = iter->rb->root; + iter->next_step = NextStepLeft; + goto loop; + + case NextStepLeft: + while (iter->last_visited->left != RBNIL) + iter->last_visited = iter->last_visited->left; + + iter->next_step = NextStepRight; + goto loop; + + case NextStepRight: + if (iter->last_visited->right != RBNIL) { - node->iteratorState = FirstStepDone; - descend(node->left); + iter->last_visited = iter->last_visited->right; + iter->next_step = NextStepLeft; + goto loop; } - /* FALL THROUGH */ - case FirstStepDone: - if (node->right != RBNIL) + else /* not moved - return current, then go up */ + iter->next_step = NextStepUp; + break; + + case NextStepUp: + for (;;) { - node->iteratorState = SecondStepDone; - descend(node->right); + came_from = iter->last_visited; + iter->last_visited = iter->last_visited->parent; + if (iter->last_visited == NULL) + { + iter->is_over = true; + break; /* end of iteration */ + } + + if (came_from == iter->last_visited->right) + { + /* return current, then continue to go up */ + break; + } + + /* otherwise we came from the left */ + iter->next_step = NextStepRight; + goto loop; } - /* FALL THROUGH */ - case SecondStepDone: - node->iteratorState = ThirdStepDone; - return node; - case ThirdStepDone: - if (node->parent) - ascend(node->parent); break; - default: - elog(ERROR, "unrecognized rbtree node state: %d", - node->iteratorState); } - return NULL; + return iter->last_visited; } /* @@ -827,33 +843,34 @@ rb_inverted_iterator(RBTree *rb) * returns NULL or the traversal stops being of interest. * * If the tree is changed during traversal, results of further calls to - * rb_iterate are unspecified. + * rb_iterate are unspecified. Multiple concurrent iterators on the same + * tree are allowed. * - * Note: this used to return a separately palloc'd iterator control struct, - * but that's a bit pointless since the data structure is incapable of - * supporting multiple concurrent traversals. Now we just keep the state - * in RBTree. + * The iterator state is stored in the 'iter' struct. The caller should + * treat it as opaque struct. */ void -rb_begin_iterate(RBTree *rb, RBOrderControl ctrl) +rb_begin_iterate(RBTree *rb, RBOrderControl ctrl, RBTreeIterator *iter) { - rb->cur = rb->root; - if (rb->cur != RBNIL) - rb->cur->iteratorState = InitialState; + /* Common initialization for all traversal orders */ + iter->rb = rb; + iter->last_visited = NULL; + iter->is_over = (rb->root == RBNIL); switch (ctrl) { case LeftRightWalk: /* visit left, then self, then right */ - rb->iterate = rb_left_right_iterator; + iter->iterate = rb_left_right_iterator; break; case RightLeftWalk: /* visit right, then self, then left */ - rb->iterate = rb_right_left_iterator; + iter->iterate = rb_right_left_iterator; break; case DirectWalk: /* visit self, then left, then right */ - rb->iterate = rb_direct_iterator; + iter->iterate = rb_direct_iterator; break; case InvertedWalk: /* visit left, then right, then self */ - rb->iterate = rb_inverted_iterator; + iter->iterate = rb_inverted_iterator; + iter->next_step = NextStepBegin; break; default: elog(ERROR, "unrecognized rbtree iteration order: %d", ctrl); @@ -864,10 +881,10 @@ rb_begin_iterate(RBTree *rb, RBOrderControl ctrl) * rb_iterate: return the next node in traversal order, or NULL if no more */ RBNode * -rb_iterate(RBTree *rb) +rb_iterate(RBTreeIterator *iter) { - if (rb->cur == RBNIL) + if (iter->is_over) return NULL; - return rb->iterate(rb); + return iter->iterate(iter); } diff --git a/src/include/access/gin_private.h b/src/include/access/gin_private.h index 68cfe0c1ac..bf589ab7a1 100644 --- a/src/include/access/gin_private.h +++ b/src/include/access/gin_private.h @@ -919,6 +919,7 @@ typedef struct GinEntryAccumulator *entryallocator; uint32 eas_used; RBTree *tree; + RBTreeIterator tree_walk; } BuildAccumulator; extern void ginInitBA(BuildAccumulator *accum); diff --git a/src/include/lib/rbtree.h b/src/include/lib/rbtree.h index 73e83c367a..4537c587ac 100644 --- a/src/include/lib/rbtree.h +++ b/src/include/lib/rbtree.h @@ -22,7 +22,6 @@ */ typedef struct RBNode { - char iteratorState; /* workspace for iterating through tree */ char color; /* node's current color, red or black */ struct RBNode *left; /* left child, or RBNIL if none */ struct RBNode *right; /* right child, or RBNIL if none */ @@ -41,6 +40,22 @@ typedef enum RBOrderControl InvertedWalk /* postorder: left child, right child, node */ } RBOrderControl; +/* + * RBTreeIterator holds state while traversing a tree. This is declared + * here so that callers can stack-allocate this, but must otherwise be + * treated as an opaque struct. + */ +typedef struct RBTreeIterator RBTreeIterator; + +struct RBTreeIterator +{ + RBTree *rb; + RBNode *(*iterate) (RBTreeIterator *iter); + RBNode *last_visited; + char next_step; + bool is_over; +}; + /* Support functions to be provided by caller */ typedef int (*rb_comparator) (const RBNode *a, const RBNode *b, void *arg); typedef void (*rb_combiner) (RBNode *existing, const RBNode *newdata, void *arg); @@ -60,7 +75,8 @@ extern RBNode *rb_leftmost(RBTree *rb); extern RBNode *rb_insert(RBTree *rb, const RBNode *data, bool *isNew); extern void rb_delete(RBTree *rb, RBNode *node); -extern void rb_begin_iterate(RBTree *rb, RBOrderControl ctrl); -extern RBNode *rb_iterate(RBTree *rb); +extern void rb_begin_iterate(RBTree *rb, RBOrderControl ctrl, + RBTreeIterator *iter); +extern RBNode *rb_iterate(RBTreeIterator *iter); #endif /* RBTREE_H */ From 9cca11c915e458323d0e746c68203f2c11da0302 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 2 Sep 2016 11:51:49 +0300 Subject: [PATCH 099/871] Speed up SUM calculation in numeric aggregates. This introduces a numeric sum accumulator, which performs better than repeatedly calling add_var(). The performance comes from using wider digits and delaying carry propagation, tallying positive and negative values separately, and avoiding a round of palloc/pfree on every value. This speeds up SUM(), as well as other standard aggregates like AVG() and STDDEV() that also calculate a sum internally. Reviewed-by: Andrey Borodin Discussion: --- src/backend/utils/adt/numeric.c | 601 +++++++++++++++++++++----- src/test/regress/expected/numeric.out | 16 + src/test/regress/sql/numeric.sql | 8 + 3 files changed, 522 insertions(+), 103 deletions(-) diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index 620226cea1..27efd310ab 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -302,6 +302,49 @@ typedef struct hyperLogLogState abbr_card; /* cardinality estimator */ } NumericSortSupport; + +/* ---------- + * Fast sum accumulator. + * + * NumericSumAccum is used to implement SUM(), and other standard aggregates + * that track the sum of input values. It uses 32-bit integers to store the + * digits, instead of the normal 16-bit integers (with NBASE=10000). This + * way, we can safely accumulate up to NBASE - 1 values without propagating + * carry, before risking overflow of any of the digits. 'num_uncarried' + * tracks how many values have been accumulated without propagating carry. + * + * Positive and negative values are accumulated separately, in 'pos_digits' + * and 'neg_digits'. This is simpler and faster than deciding whether to add + * or subtract from the current value, for each new value (see sub_var() for + * the logic we avoid by doing this). Both buffers are of same size, and + * have the same weight and scale. In accum_sum_final(), the positive and + * negative sums are added together to produce the final result. + * + * When a new value has a larger ndigits or weight than the accumulator + * currently does, the accumulator is enlarged to accommodate the new value. + * We normally have one zero digit reserved for carry propagation, and that + * is indicated by the 'have_carry_space' flag. When accum_sum_carry() uses + * up the reserved digit, it clears the 'have_carry_space' flag. The next + * call to accum_sum_add() will enlarge the buffer, to make room for the + * extra digit, and set the flag again. + * + * To initialize a new accumulator, simply reset all fields to zeros. + * + * The accumulator does not handle NaNs. + * ---------- + */ +typedef struct NumericSumAccum +{ + int ndigits; + int weight; + int dscale; + int num_uncarried; + bool have_carry_space; + int32 *pos_digits; + int32 *neg_digits; +} NumericSumAccum; + + /* * We define our own macros for packing and unpacking abbreviated-key * representations for numeric values in order to avoid depending on @@ -490,6 +533,14 @@ static void strip_var(NumericVar *var); static void compute_bucket(Numeric operand, Numeric bound1, Numeric bound2, NumericVar *count_var, NumericVar *result_var); +static void accum_sum_add(NumericSumAccum *accum, NumericVar *var1); +static void accum_sum_rescale(NumericSumAccum *accum, NumericVar *val); +static void accum_sum_carry(NumericSumAccum *accum); +static void accum_sum_reset(NumericSumAccum *accum); +static void accum_sum_final(NumericSumAccum *accum, NumericVar *result); +static void accum_sum_copy(NumericSumAccum *dst, NumericSumAccum *src); +static void accum_sum_combine(NumericSumAccum *accum, NumericSumAccum *accum2); + /* ---------------------------------------------------------------------- * @@ -3140,8 +3191,8 @@ typedef struct NumericAggState bool calcSumX2; /* if true, calculate sumX2 */ MemoryContext agg_context; /* context we're calculating in */ int64 N; /* count of processed numbers */ - NumericVar sumX; /* sum of processed numbers */ - NumericVar sumX2; /* sum of squares of processed numbers */ + NumericSumAccum sumX; /* sum of processed numbers */ + NumericSumAccum sumX2; /* sum of squares of processed numbers */ int maxScale; /* maximum scale seen so far */ int64 maxScaleCount; /* number of values seen with maximum scale */ int64 NaNcount; /* count of NaN values (not included in N!) */ @@ -3230,22 +3281,13 @@ do_numeric_accum(NumericAggState *state, Numeric newval) /* The rest of this needs to work in the aggregate context */ old_context = MemoryContextSwitchTo(state->agg_context); - if (state->N++ > 0) - { - /* Accumulate sums */ - add_var(&X, &(state->sumX), &(state->sumX)); + state->N++; - if (state->calcSumX2) - add_var(&X2, &(state->sumX2), &(state->sumX2)); - } - else - { - /* First input, so initialize sums */ - set_var_from_var(&X, &(state->sumX)); + /* Accumulate sums */ + accum_sum_add(&(state->sumX), &X); - if (state->calcSumX2) - set_var_from_var(&X2, &(state->sumX2)); - } + if (state->calcSumX2) + accum_sum_add(&(state->sumX2), &X2); MemoryContextSwitchTo(old_context); } @@ -3324,16 +3366,25 @@ do_numeric_discard(NumericAggState *state, Numeric newval) if (state->N-- > 1) { - /* De-accumulate sums */ - sub_var(&(state->sumX), &X, &(state->sumX)); + /* Negate X, to subtract it from the sum */ + X.sign = (X.sign == NUMERIC_POS ? NUMERIC_NEG : NUMERIC_POS); + accum_sum_add(&(state->sumX), &X); if (state->calcSumX2) - sub_var(&(state->sumX2), &X2, &(state->sumX2)); + { + /* Negate X^2. X^2 is always positive */ + X2.sign = NUMERIC_NEG; + accum_sum_add(&(state->sumX2), &X2); + } } else { - /* Sums will be reset by next call to do_numeric_accum */ + /* Zero the sums */ Assert(state->N == 0); + + accum_sum_reset(&state->sumX); + if (state->calcSumX2) + accum_sum_reset(&state->sumX2); } MemoryContextSwitchTo(old_context); @@ -3392,11 +3443,8 @@ numeric_combine(PG_FUNCTION_ARGS) state1->maxScale = state2->maxScale; state1->maxScaleCount = state2->maxScaleCount; - init_var(&state1->sumX); - set_var_from_var(&state2->sumX, &state1->sumX); - - init_var(&state1->sumX2); - set_var_from_var(&state2->sumX2, &state1->sumX2); + accum_sum_copy(&state1->sumX, &state2->sumX); + accum_sum_copy(&state1->sumX2, &state2->sumX2); MemoryContextSwitchTo(old_context); @@ -3424,8 +3472,8 @@ numeric_combine(PG_FUNCTION_ARGS) old_context = MemoryContextSwitchTo(agg_context); /* Accumulate sums */ - add_var(&(state1->sumX), &(state2->sumX), &(state1->sumX)); - add_var(&(state1->sumX2), &(state2->sumX2), &(state1->sumX2)); + accum_sum_combine(&state1->sumX, &state2->sumX); + accum_sum_combine(&state1->sumX2, &state2->sumX2); MemoryContextSwitchTo(old_context); } @@ -3483,8 +3531,7 @@ numeric_avg_combine(PG_FUNCTION_ARGS) state1->maxScale = state2->maxScale; state1->maxScaleCount = state2->maxScaleCount; - init_var(&state1->sumX); - set_var_from_var(&state2->sumX, &state1->sumX); + accum_sum_copy(&state1->sumX, &state2->sumX); MemoryContextSwitchTo(old_context); @@ -3512,7 +3559,7 @@ numeric_avg_combine(PG_FUNCTION_ARGS) old_context = MemoryContextSwitchTo(agg_context); /* Accumulate sums */ - add_var(&(state1->sumX), &(state2->sumX), &(state1->sumX)); + accum_sum_combine(&state1->sumX, &state2->sumX); MemoryContextSwitchTo(old_context); } @@ -3532,6 +3579,7 @@ numeric_avg_serialize(PG_FUNCTION_ARGS) Datum temp; bytea *sumX; bytea *result; + NumericVar tmp_var; /* Ensure we disallow calling when not in aggregate context */ if (!AggCheckCallContext(fcinfo, NULL)) @@ -3545,9 +3593,13 @@ numeric_avg_serialize(PG_FUNCTION_ARGS) * splitting the tasks in numeric_send into separate functions to stop * this? Doing so would also remove the fmgr call overhead. */ + init_var(&tmp_var); + accum_sum_final(&state->sumX, &tmp_var); + temp = DirectFunctionCall1(numeric_send, - NumericGetDatum(make_result(&state->sumX))); + NumericGetDatum(make_result(&tmp_var))); sumX = DatumGetByteaP(temp); + free_var(&tmp_var); pq_begintypsend(&buf); @@ -3582,6 +3634,7 @@ numeric_avg_deserialize(PG_FUNCTION_ARGS) bytea *sstate; NumericAggState *result; Datum temp; + NumericVar tmp_var; StringInfoData buf; if (!AggCheckCallContext(fcinfo, NULL)) @@ -3606,7 +3659,8 @@ numeric_avg_deserialize(PG_FUNCTION_ARGS) PointerGetDatum(&buf), InvalidOid, -1); - set_var_from_num(DatumGetNumeric(temp), &result->sumX); + init_var_from_num(DatumGetNumeric(temp), &tmp_var); + accum_sum_add(&(result->sumX), &tmp_var); /* maxScale */ result->maxScale = pq_getmsgint(&buf, 4); @@ -3635,6 +3689,7 @@ numeric_serialize(PG_FUNCTION_ARGS) StringInfoData buf; Datum temp; bytea *sumX; + NumericVar tmp_var; bytea *sumX2; bytea *result; @@ -3650,14 +3705,20 @@ numeric_serialize(PG_FUNCTION_ARGS) * splitting the tasks in numeric_send into separate functions to stop * this? Doing so would also remove the fmgr call overhead. */ + init_var(&tmp_var); + + accum_sum_final(&state->sumX, &tmp_var); temp = DirectFunctionCall1(numeric_send, - NumericGetDatum(make_result(&state->sumX))); + NumericGetDatum(make_result(&tmp_var))); sumX = DatumGetByteaP(temp); + accum_sum_final(&state->sumX2, &tmp_var); temp = DirectFunctionCall1(numeric_send, - NumericGetDatum(make_result(&state->sumX2))); + NumericGetDatum(make_result(&tmp_var))); sumX2 = DatumGetByteaP(temp); + free_var(&tmp_var); + pq_begintypsend(&buf); /* N */ @@ -3694,6 +3755,8 @@ numeric_deserialize(PG_FUNCTION_ARGS) bytea *sstate; NumericAggState *result; Datum temp; + NumericVar sumX_var; + NumericVar sumX2_var; StringInfoData buf; if (!AggCheckCallContext(fcinfo, NULL)) @@ -3718,14 +3781,16 @@ numeric_deserialize(PG_FUNCTION_ARGS) PointerGetDatum(&buf), InvalidOid, -1); - set_var_from_num(DatumGetNumeric(temp), &result->sumX); + init_var_from_num(DatumGetNumeric(temp), &sumX_var); + accum_sum_add(&(result->sumX), &sumX_var); /* sumX2 */ temp = DirectFunctionCall3(numeric_recv, PointerGetDatum(&buf), InvalidOid, -1); - set_var_from_num(DatumGetNumeric(temp), &result->sumX2); + init_var_from_num(DatumGetNumeric(temp), &sumX2_var); + accum_sum_add(&(result->sumX2), &sumX2_var); /* maxScale */ result->maxScale = pq_getmsgint(&buf, 4); @@ -3974,11 +4039,8 @@ numeric_poly_combine(PG_FUNCTION_ARGS) state1->sumX = state2->sumX; state1->sumX2 = state2->sumX2; #else - init_var(&(state1->sumX)); - set_var_from_var(&(state2->sumX), &(state1->sumX)); - - init_var(&state1->sumX2); - set_var_from_var(&(state2->sumX2), &(state1->sumX2)); + accum_sum_copy(&state2->sumX, &state1->sumX); + accum_sum_copy(&state2->sumX2, &state1->sumX2); #endif MemoryContextSwitchTo(old_context); @@ -3998,8 +4060,8 @@ numeric_poly_combine(PG_FUNCTION_ARGS) old_context = MemoryContextSwitchTo(agg_context); /* Accumulate sums */ - add_var(&(state1->sumX), &(state2->sumX), &(state1->sumX)); - add_var(&(state1->sumX2), &(state2->sumX2), &(state1->sumX2)); + accum_sum_combine(&state1->sumX, &state2->sumX); + accum_sum_combine(&state1->sumX2, &state2->sumX2); MemoryContextSwitchTo(old_context); #endif @@ -4038,30 +4100,29 @@ numeric_poly_serialize(PG_FUNCTION_ARGS) */ { Datum temp; - -#ifdef HAVE_INT128 NumericVar num; init_var(&num); + +#ifdef HAVE_INT128 int128_to_numericvar(state->sumX, &num); +#else + accum_sum_final(&state->sumX, &num); +#endif temp = DirectFunctionCall1(numeric_send, NumericGetDatum(make_result(&num))); sumX = DatumGetByteaP(temp); +#ifdef HAVE_INT128 int128_to_numericvar(state->sumX2, &num); +#else + accum_sum_final(&state->sumX2, &num); +#endif temp = DirectFunctionCall1(numeric_send, NumericGetDatum(make_result(&num))); sumX2 = DatumGetByteaP(temp); - free_var(&num); -#else - temp = DirectFunctionCall1(numeric_send, - NumericGetDatum(make_result(&state->sumX))); - sumX = DatumGetByteaP(temp); - temp = DirectFunctionCall1(numeric_send, - NumericGetDatum(make_result(&state->sumX2))); - sumX2 = DatumGetByteaP(temp); -#endif + free_var(&num); } pq_begintypsend(&buf); @@ -4091,7 +4152,9 @@ numeric_poly_deserialize(PG_FUNCTION_ARGS) bytea *sstate; PolyNumAggState *result; Datum sumX; + NumericVar sumX_var; Datum sumX2; + NumericVar sumX2_var; StringInfoData buf; if (!AggCheckCallContext(fcinfo, NULL)) @@ -4123,22 +4186,18 @@ numeric_poly_deserialize(PG_FUNCTION_ARGS) InvalidOid, -1); + init_var_from_num(DatumGetNumeric(sumX), &sumX_var); #ifdef HAVE_INT128 - { - NumericVar num; - - init_var(&num); - set_var_from_num(DatumGetNumeric(sumX), &num); - numericvar_to_int128(&num, &result->sumX); - - set_var_from_num(DatumGetNumeric(sumX2), &num); - numericvar_to_int128(&num, &result->sumX2); + numericvar_to_int128(&sumX_var, &result->sumX); +#else + accum_sum_add(&result->sumX, &sumX_var); +#endif - free_var(&num); - } + set_var_from_num(DatumGetNumeric(sumX2), &sumX2_var); +#ifdef HAVE_INT128 + numericvar_to_int128(&sumX2_var, &result->sumX2); #else - set_var_from_num(DatumGetNumeric(sumX), &result->sumX); - set_var_from_num(DatumGetNumeric(sumX2), &result->sumX2); + accum_sum_add(&result->sumX2, &sumX_var); #endif pq_getmsgend(&buf); @@ -4209,8 +4268,7 @@ int8_avg_combine(PG_FUNCTION_ARGS) #ifdef HAVE_INT128 state1->sumX = state2->sumX; #else - init_var(&state1->sumX); - set_var_from_var(&state2->sumX, &state1->sumX); + accum_sum_copy(&state1->sumX, &state2->sumX); #endif MemoryContextSwitchTo(old_context); @@ -4228,7 +4286,7 @@ int8_avg_combine(PG_FUNCTION_ARGS) old_context = MemoryContextSwitchTo(agg_context); /* Accumulate sums */ - add_var(&(state1->sumX), &(state2->sumX), &(state1->sumX)); + accum_sum_combine(&state1->sumX, &state2->sumX); MemoryContextSwitchTo(old_context); #endif @@ -4266,20 +4324,20 @@ int8_avg_serialize(PG_FUNCTION_ARGS) */ { Datum temp; -#ifdef HAVE_INT128 NumericVar num; init_var(&num); + +#ifdef HAVE_INT128 int128_to_numericvar(state->sumX, &num); - temp = DirectFunctionCall1(numeric_send, - NumericGetDatum(make_result(&num))); - free_var(&num); - sumX = DatumGetByteaP(temp); #else + accum_sum_final(&state->sumX, &num); +#endif temp = DirectFunctionCall1(numeric_send, - NumericGetDatum(make_result(&state->sumX))); + NumericGetDatum(make_result(&num))); sumX = DatumGetByteaP(temp); -#endif + + free_var(&num); } pq_begintypsend(&buf); @@ -4306,6 +4364,7 @@ int8_avg_deserialize(PG_FUNCTION_ARGS) PolyNumAggState *result; StringInfoData buf; Datum temp; + NumericVar num; if (!AggCheckCallContext(fcinfo, NULL)) elog(ERROR, "aggregate function called in non-aggregate context"); @@ -4329,18 +4388,11 @@ int8_avg_deserialize(PG_FUNCTION_ARGS) PointerGetDatum(&buf), InvalidOid, -1); - + init_var_from_num(DatumGetNumeric(temp), &num); #ifdef HAVE_INT128 - { - NumericVar num; - - init_var(&num); - set_var_from_num(DatumGetNumeric(temp), &num); - numericvar_to_int128(&num, &result->sumX); - free_var(&num); - } + numericvar_to_int128(&num, &result->sumX); #else - set_var_from_num(DatumGetNumeric(temp), &result->sumX); + accum_sum_add(&result->sumX, &num); #endif pq_getmsgend(&buf); @@ -4534,6 +4586,7 @@ numeric_avg(PG_FUNCTION_ARGS) NumericAggState *state; Datum N_datum; Datum sumX_datum; + NumericVar sumX_var; state = PG_ARGISNULL(0) ? NULL : (NumericAggState *) PG_GETARG_POINTER(0); @@ -4545,7 +4598,11 @@ numeric_avg(PG_FUNCTION_ARGS) PG_RETURN_NUMERIC(make_result(&const_nan)); N_datum = DirectFunctionCall1(int8_numeric, Int64GetDatum(state->N)); - sumX_datum = NumericGetDatum(make_result(&state->sumX)); + + init_var(&sumX_var); + accum_sum_final(&state->sumX, &sumX_var); + sumX_datum = NumericGetDatum(make_result(&sumX_var)); + free_var(&sumX_var); PG_RETURN_DATUM(DirectFunctionCall2(numeric_div, sumX_datum, N_datum)); } @@ -4554,6 +4611,8 @@ Datum numeric_sum(PG_FUNCTION_ARGS) { NumericAggState *state; + NumericVar sumX_var; + Numeric result; state = PG_ARGISNULL(0) ? NULL : (NumericAggState *) PG_GETARG_POINTER(0); @@ -4564,7 +4623,12 @@ numeric_sum(PG_FUNCTION_ARGS) if (state->NaNcount > 0) /* there was at least one NaN input */ PG_RETURN_NUMERIC(make_result(&const_nan)); - PG_RETURN_NUMERIC(make_result(&(state->sumX))); + init_var(&sumX_var); + accum_sum_final(&state->sumX, &sumX_var); + result = make_result(&sumX_var); + free_var(&sumX_var); + + PG_RETURN_NUMERIC(result); } /* @@ -4608,8 +4672,8 @@ numeric_stddev_internal(NumericAggState *state, init_var(&vsumX2); int64_to_numericvar(state->N, &vN); - set_var_from_var(&(state->sumX), &vsumX); - set_var_from_var(&(state->sumX2), &vsumX2); + accum_sum_final(&(state->sumX), &vsumX); + accum_sum_final(&(state->sumX2), &vsumX2); /* * Sample stddev and variance are undefined when N <= 1; population stddev @@ -4739,26 +4803,38 @@ numeric_poly_stddev_internal(Int128AggState *state, NumericAggState numstate; Numeric res; - init_var(&numstate.sumX); - init_var(&numstate.sumX2); - numstate.NaNcount = 0; - numstate.agg_context = NULL; + /* Initialize an empty agg state */ + memset(&numstate, 0, sizeof(NumericAggState)); if (state) { + NumericVar tmp_var; + numstate.N = state->N; - int128_to_numericvar(state->sumX, &numstate.sumX); - int128_to_numericvar(state->sumX2, &numstate.sumX2); - } - else - { - numstate.N = 0; + + init_var(&tmp_var); + + int128_to_numericvar(state->sumX, &tmp_var); + accum_sum_add(&numstate.sumX, &tmp_var); + + int128_to_numericvar(state->sumX2, &tmp_var); + accum_sum_add(&numstate.sumX2, &tmp_var); + + free_var(&tmp_var); } res = numeric_stddev_internal(&numstate, variance, sample, is_null); - free_var(&numstate.sumX); - free_var(&numstate.sumX2); + if (numstate.sumX.ndigits > 0) + { + pfree(numstate.sumX.pos_digits); + pfree(numstate.sumX.neg_digits); + } + if (numstate.sumX2.ndigits > 0) + { + pfree(numstate.sumX2.pos_digits); + pfree(numstate.sumX2.neg_digits); + } return res; } @@ -8702,3 +8778,322 @@ strip_var(NumericVar *var) var->digits = digits; var->ndigits = ndigits; } + + +/* ---------------------------------------------------------------------- + * + * Fast sum accumulator functions + * + * ---------------------------------------------------------------------- + */ + +/* + * Reset the accumulator's value to zero. The buffers to hold the digits + * are not free'd. + */ +static void +accum_sum_reset(NumericSumAccum *accum) +{ + int i; + + accum->dscale = 0; + for (i = 0; i < accum->ndigits; i++) + { + accum->pos_digits[i] = 0; + accum->neg_digits[i] = 0; + } +} + +/* + * Accumulate a new value. + */ +static void +accum_sum_add(NumericSumAccum *accum, NumericVar *val) +{ + int32 *accum_digits; + int i, + val_i; + int val_ndigits; + NumericDigit *val_digits; + + /* + * If we have accumulated too many values since the last carry + * propagation, do it now, to avoid overflowing. (We could allow more + * than NBASE - 1, if we reserved two extra digits, rather than one, for + * carry propagation. But even with NBASE - 1, this needs to be done so + * seldom, that the performance difference is negligible.) + */ + if (accum->num_uncarried == NBASE - 1) + accum_sum_carry(accum); + + /* + * Adjust the weight or scale of the old value, so that it can accommodate + * the new value. + */ + accum_sum_rescale(accum, val); + + /* */ + if (val->sign == NUMERIC_POS) + accum_digits = accum->pos_digits; + else + accum_digits = accum->neg_digits; + + /* copy these values into local vars for speed in loop */ + val_ndigits = val->ndigits; + val_digits = val->digits; + + i = accum->weight - val->weight; + for (val_i = 0; val_i < val_ndigits; val_i++) + { + accum_digits[i] += (int32) val_digits[val_i]; + i++; + } + + accum->num_uncarried++; +} + +/* + * Propagate carries. + */ +static void +accum_sum_carry(NumericSumAccum *accum) +{ + int i; + int ndigits; + int32 *dig; + int32 carry; + int32 newdig = 0; + + /* + * If no new values have been added since last carry propagation, nothing + * to do. + */ + if (accum->num_uncarried == 0) + return; + + /* + * We maintain that the weight of the accumulator is always one larger + * than needed to hold the current value, before carrying, to make sure + * there is enough space for the possible extra digit when carry is + * propagated. We cannot expand the buffer here, unless we require + * callers of accum_sum_final() to switch to the right memory context. + */ + Assert(accum->pos_digits[0] == 0 && accum->neg_digits[0] == 0); + + ndigits = accum->ndigits; + + /* Propagate carry in the positive sum */ + dig = accum->pos_digits; + carry = 0; + for (i = ndigits - 1; i >= 0; i--) + { + newdig = dig[i] + carry; + if (newdig >= NBASE) + { + carry = newdig / NBASE; + newdig -= carry * NBASE; + } + else + carry = 0; + dig[i] = newdig; + } + /* Did we use up the digit reserved for carry propagation? */ + if (newdig > 0) + accum->have_carry_space = false; + + /* And the same for the negative sum */ + dig = accum->neg_digits; + carry = 0; + for (i = ndigits - 1; i >= 0; i--) + { + newdig = dig[i] + carry; + if (newdig >= NBASE) + { + carry = newdig / NBASE; + newdig -= carry * NBASE; + } + else + carry = 0; + dig[i] = newdig; + } + if (newdig > 0) + accum->have_carry_space = false; + + accum->num_uncarried = 0; +} + +/* + * Re-scale accumulator to accommodate new value. + * + * If the new value has more digits than the current digit buffers in the + * accumulator, enlarge the buffers. + */ +static void +accum_sum_rescale(NumericSumAccum *accum, NumericVar *val) +{ + int old_weight = accum->weight; + int old_ndigits = accum->ndigits; + int accum_ndigits; + int accum_weight; + int accum_rscale; + int val_rscale; + + accum_weight = old_weight; + accum_ndigits = old_ndigits; + + /* + * Does the new value have a larger weight? If so, enlarge the buffers, + * and shift the existing value to the new weight, by adding leading + * zeros. + * + * We enforce that the accumulator always has a weight one larger than + * needed for the inputs, so that we have space for an extra digit at the + * final carry-propagation phase, if necessary. + */ + if (val->weight >= accum_weight) + { + accum_weight = val->weight + 1; + accum_ndigits = accum_ndigits + (accum_weight - old_weight); + } + + /* + * Even though the new value is small, we might've used up the space + * reserved for the carry digit in the last call to accum_sum_carry(). If + * so, enlarge to make room for another one. + */ + else if (!accum->have_carry_space) + { + accum_weight++; + accum_ndigits++; + } + + /* Is the new value wider on the right side? */ + accum_rscale = accum_ndigits - accum_weight - 1; + val_rscale = val->ndigits - val->weight - 1; + if (val_rscale > accum_rscale) + accum_ndigits = accum_ndigits + (val_rscale - accum_rscale); + + if (accum_ndigits != old_ndigits || + accum_weight != old_weight) + { + int32 *new_pos_digits; + int32 *new_neg_digits; + int weightdiff; + + weightdiff = accum_weight - old_weight; + + new_pos_digits = palloc0(accum_ndigits * sizeof(int32)); + new_neg_digits = palloc0(accum_ndigits * sizeof(int32)); + + if (accum->pos_digits) + { + memcpy(&new_pos_digits[weightdiff], accum->pos_digits, + old_ndigits * sizeof(int32)); + pfree(accum->pos_digits); + + memcpy(&new_neg_digits[weightdiff], accum->neg_digits, + old_ndigits * sizeof(int32)); + pfree(accum->neg_digits); + } + + accum->pos_digits = new_pos_digits; + accum->neg_digits = new_neg_digits; + + accum->weight = accum_weight; + accum->ndigits = accum_ndigits; + + Assert(accum->pos_digits[0] == 0 && accum->neg_digits[0] == 0); + accum->have_carry_space = true; + } + + if (val->dscale > accum->dscale) + accum->dscale = val->dscale; +} + +/* + * Return the current value of the accumulator. This perform final carry + * propagation, and adds together the positive and negative sums. + * + * Unlike all the other routines, the caller is not required to switch to + * the memory context that holds the accumulator. + */ +static void +accum_sum_final(NumericSumAccum *accum, NumericVar *result) +{ + int i; + NumericVar pos_var; + NumericVar neg_var; + + if (accum->ndigits == 0) + { + set_var_from_var(&const_zero, result); + return; + } + + /* Perform final carry */ + accum_sum_carry(accum); + + /* Create NumericVars representing the positive and negative sums */ + init_var(&pos_var); + init_var(&neg_var); + + pos_var.ndigits = neg_var.ndigits = accum->ndigits; + pos_var.weight = neg_var.weight = accum->weight; + pos_var.dscale = neg_var.dscale = accum->dscale; + pos_var.sign = NUMERIC_POS; + neg_var.sign = NUMERIC_NEG; + + pos_var.buf = pos_var.digits = digitbuf_alloc(accum->ndigits); + neg_var.buf = neg_var.digits = digitbuf_alloc(accum->ndigits); + + for (i = 0; i < accum->ndigits; i++) + { + Assert(accum->pos_digits[i] < NBASE); + pos_var.digits[i] = (int16) accum->pos_digits[i]; + + Assert(accum->neg_digits[i] < NBASE); + neg_var.digits[i] = (int16) accum->neg_digits[i]; + } + + /* And add them together */ + add_var(&pos_var, &neg_var, result); + + /* Remove leading/trailing zeroes */ + strip_var(result); +} + +/* + * Copy an accumulator's state. + * + * 'dst' is assumed to be uninitialized beforehand. No attempt is made at + * freeing old values. + */ +static void +accum_sum_copy(NumericSumAccum *dst, NumericSumAccum *src) +{ + dst->pos_digits = palloc(src->ndigits * sizeof(int32)); + dst->neg_digits = palloc(src->ndigits * sizeof(int32)); + + memcpy(dst->pos_digits, src->pos_digits, src->ndigits * sizeof(int32)); + memcpy(dst->neg_digits, src->neg_digits, src->ndigits * sizeof(int32)); + dst->num_uncarried = src->num_uncarried; + dst->ndigits = src->ndigits; + dst->weight = src->weight; + dst->dscale = src->dscale; +} + +/* + * Add the current value of 'accum2' into 'accum'. + */ +static void +accum_sum_combine(NumericSumAccum *accum, NumericSumAccum *accum2) +{ + NumericVar tmp_var; + + init_var(&tmp_var); + + accum_sum_final(accum2, &tmp_var); + accum_sum_add(accum, &tmp_var); + + free_var(&tmp_var); +} diff --git a/src/test/regress/expected/numeric.out b/src/test/regress/expected/numeric.out index f1f50560ee..ae0beb9b68 100644 --- a/src/test/regress/expected/numeric.out +++ b/src/test/regress/expected/numeric.out @@ -1909,3 +1909,19 @@ select scale(-13.000000000000000); 15 (1 row) +-- +-- Tests for SUM() +-- +-- cases that need carry propagation +SELECT SUM(9999::numeric) FROM generate_series(1, 100000); + sum +----------- + 999900000 +(1 row) + +SELECT SUM((-9999)::numeric) FROM generate_series(1, 100000); + sum +------------ + -999900000 +(1 row) + diff --git a/src/test/regress/sql/numeric.sql b/src/test/regress/sql/numeric.sql index fc472187d8..b51225c47f 100644 --- a/src/test/regress/sql/numeric.sql +++ b/src/test/regress/sql/numeric.sql @@ -997,3 +997,11 @@ select scale(1.12345); select scale(110123.12475871856128); select scale(-1123.12471856128); select scale(-13.000000000000000); + +-- +-- Tests for SUM() +-- + +-- cases that need carry propagation +SELECT SUM(9999::numeric) FROM generate_series(1, 100000); +SELECT SUM((-9999)::numeric) FROM generate_series(1, 100000); From ec136d19b21791c845b1deeff43df137add0639e Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 2 Sep 2016 13:49:59 +0300 Subject: [PATCH 100/871] Move code shared between libpq and backend from backend/libpq/ to common/. When building libpq, ip.c and md5.c were symlinked or copied from src/backend/libpq into src/interfaces/libpq, but now that we have a directory specifically for routines that are shared between the server and client binaries, src/common/, move them there. Some routines in ip.c were only used in the backend. Keep those in src/backend/libpq, but rename to ifaddr.c to avoid confusion with the file that's now in common. Fix the comment in src/common/Makefile to reflect how libpq actually links those files. There are two more files that libpq symlinks directly from src/backend: encnames.c and wchar.c. I don't feel compelled to move those right now, though. Patch by Michael Paquier, with some changes by me. Discussion: <69938195-9c76-8523-0af8-eb718ea5b36e@iki.fi> --- contrib/passwordcheck/passwordcheck.c | 2 +- src/backend/commands/user.c | 2 +- src/backend/libpq/Makefile | 2 +- src/backend/libpq/auth.c | 4 +- src/backend/libpq/crypt.c | 2 +- src/backend/libpq/hba.c | 3 +- src/backend/libpq/{ip.c => ifaddr.c} | 232 +---------------------- src/backend/libpq/pqcomm.c | 2 +- src/backend/postmaster/pgstat.c | 2 +- src/backend/postmaster/postmaster.c | 2 +- src/backend/utils/adt/network.c | 2 +- src/backend/utils/adt/pgstatfuncs.c | 2 +- src/backend/utils/adt/varlena.c | 2 +- src/common/Makefile | 10 +- src/common/ip.c | 260 ++++++++++++++++++++++++++ src/{backend/libpq => common}/md5.c | 11 +- src/include/{libpq => common}/ip.h | 18 +- src/include/{libpq => common}/md5.h | 2 +- src/include/libpq/ifaddr.h | 30 +++ src/interfaces/libpq/Makefile | 8 +- src/interfaces/libpq/fe-auth.c | 2 +- src/interfaces/libpq/fe-connect.c | 2 +- src/tools/ifaddrs/Makefile | 4 +- src/tools/ifaddrs/test_ifaddrs.c | 2 +- src/tools/msvc/Mkvcbuild.pm | 4 +- 25 files changed, 337 insertions(+), 275 deletions(-) rename src/backend/libpq/{ip.c => ifaddr.c} (72%) create mode 100644 src/common/ip.c rename src/{backend/libpq => common}/md5.c (98%) rename src/include/{libpq => common}/ip.h (65%) rename src/include/{libpq => common}/md5.h (96%) create mode 100644 src/include/libpq/ifaddr.h diff --git a/contrib/passwordcheck/passwordcheck.c b/contrib/passwordcheck/passwordcheck.c index b4c1ce005b..a0db89bbbf 100644 --- a/contrib/passwordcheck/passwordcheck.c +++ b/contrib/passwordcheck/passwordcheck.c @@ -21,8 +21,8 @@ #endif #include "commands/user.h" +#include "common/md5.h" #include "fmgr.h" -#include "libpq/md5.h" PG_MODULE_MAGIC; diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index b6ea95061d..821dce3ce7 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -29,7 +29,7 @@ #include "commands/dbcommands.h" #include "commands/seclabel.h" #include "commands/user.h" -#include "libpq/md5.h" +#include "common/md5.h" #include "miscadmin.h" #include "storage/lmgr.h" #include "utils/acl.h" diff --git a/src/backend/libpq/Makefile b/src/backend/libpq/Makefile index 09410c4bb1..1bdd8adde2 100644 --- a/src/backend/libpq/Makefile +++ b/src/backend/libpq/Makefile @@ -14,7 +14,7 @@ include $(top_builddir)/src/Makefile.global # be-fsstubs is here for historical reasons, probably belongs elsewhere -OBJS = be-fsstubs.o be-secure.o auth.o crypt.o hba.o ip.o md5.o pqcomm.o \ +OBJS = be-fsstubs.o be-secure.o auth.o crypt.o hba.o ifaddr.o pqcomm.o \ pqformat.o pqmq.o pqsignal.o ifeq ($(with_openssl),yes) diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index fc8b99b444..d907e6bc91 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -21,12 +21,12 @@ #include #include +#include "common/ip.h" +#include "common/md5.h" #include "libpq/auth.h" #include "libpq/crypt.h" -#include "libpq/ip.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" -#include "libpq/md5.h" #include "miscadmin.h" #include "replication/walsender.h" #include "storage/ipc.h" diff --git a/src/backend/libpq/crypt.c b/src/backend/libpq/crypt.c index d79f5a2496..d84a180330 100644 --- a/src/backend/libpq/crypt.c +++ b/src/backend/libpq/crypt.c @@ -21,8 +21,8 @@ #endif #include "catalog/pg_authid.h" +#include "common/md5.h" #include "libpq/crypt.h" -#include "libpq/md5.h" #include "miscadmin.h" #include "utils/builtins.h" #include "utils/syscache.h" diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index d612c11159..f1e9a38c92 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -26,7 +26,8 @@ #include #include "catalog/pg_collation.h" -#include "libpq/ip.h" +#include "common/ip.h" +#include "libpq/ifaddr.h" #include "libpq/libpq.h" #include "postmaster/postmaster.h" #include "regex/regex.h" diff --git a/src/backend/libpq/ip.c b/src/backend/libpq/ifaddr.c similarity index 72% rename from src/backend/libpq/ip.c rename to src/backend/libpq/ifaddr.c index 9591ed2862..8f7fa2015d 100644 --- a/src/backend/libpq/ip.c +++ b/src/backend/libpq/ifaddr.c @@ -1,14 +1,14 @@ /*------------------------------------------------------------------------- * - * ip.c - * IPv6-aware network access. + * ifaddr.c + * IP netmask calculations, and enumerating network interfaces. * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * * IDENTIFICATION - * src/backend/libpq/ip.c + * src/backend/libpq/ifaddr.c * * This file and the IPV6 implementation were initially provided by * Nigel Kukard , Linux Based Systems Design @@ -17,8 +17,7 @@ *------------------------------------------------------------------------- */ -/* This is intended to be used in both frontend and backend, so use c.h */ -#include "c.h" +#include "postgres.h" #include #include @@ -32,8 +31,7 @@ #include #include -#include "libpq/ip.h" - +#include "libpq/ifaddr.h" static int range_sockaddr_AF_INET(const struct sockaddr_in * addr, const struct sockaddr_in * netaddr, @@ -45,226 +43,6 @@ static int range_sockaddr_AF_INET6(const struct sockaddr_in6 * addr, const struct sockaddr_in6 * netmask); #endif -#ifdef HAVE_UNIX_SOCKETS -static int getaddrinfo_unix(const char *path, - const struct addrinfo * hintsp, - struct addrinfo ** result); - -static int getnameinfo_unix(const struct sockaddr_un * sa, int salen, - char *node, int nodelen, - char *service, int servicelen, - int flags); -#endif - - -/* - * pg_getaddrinfo_all - get address info for Unix, IPv4 and IPv6 sockets - */ -int -pg_getaddrinfo_all(const char *hostname, const char *servname, - const struct addrinfo * hintp, struct addrinfo ** result) -{ - int rc; - - /* not all versions of getaddrinfo() zero *result on failure */ - *result = NULL; - -#ifdef HAVE_UNIX_SOCKETS - if (hintp->ai_family == AF_UNIX) - return getaddrinfo_unix(servname, hintp, result); -#endif - - /* NULL has special meaning to getaddrinfo(). */ - rc = getaddrinfo((!hostname || hostname[0] == '\0') ? NULL : hostname, - servname, hintp, result); - - return rc; -} - - -/* - * pg_freeaddrinfo_all - free addrinfo structures for IPv4, IPv6, or Unix - * - * Note: the ai_family field of the original hint structure must be passed - * so that we can tell whether the addrinfo struct was built by the system's - * getaddrinfo() routine or our own getaddrinfo_unix() routine. Some versions - * of getaddrinfo() might be willing to return AF_UNIX addresses, so it's - * not safe to look at ai_family in the addrinfo itself. - */ -void -pg_freeaddrinfo_all(int hint_ai_family, struct addrinfo * ai) -{ -#ifdef HAVE_UNIX_SOCKETS - if (hint_ai_family == AF_UNIX) - { - /* struct was built by getaddrinfo_unix (see pg_getaddrinfo_all) */ - while (ai != NULL) - { - struct addrinfo *p = ai; - - ai = ai->ai_next; - free(p->ai_addr); - free(p); - } - } - else -#endif /* HAVE_UNIX_SOCKETS */ - { - /* struct was built by getaddrinfo() */ - if (ai != NULL) - freeaddrinfo(ai); - } -} - - -/* - * pg_getnameinfo_all - get name info for Unix, IPv4 and IPv6 sockets - * - * The API of this routine differs from the standard getnameinfo() definition - * in two ways: first, the addr parameter is declared as sockaddr_storage - * rather than struct sockaddr, and second, the node and service fields are - * guaranteed to be filled with something even on failure return. - */ -int -pg_getnameinfo_all(const struct sockaddr_storage * addr, int salen, - char *node, int nodelen, - char *service, int servicelen, - int flags) -{ - int rc; - -#ifdef HAVE_UNIX_SOCKETS - if (addr && addr->ss_family == AF_UNIX) - rc = getnameinfo_unix((const struct sockaddr_un *) addr, salen, - node, nodelen, - service, servicelen, - flags); - else -#endif - rc = getnameinfo((const struct sockaddr *) addr, salen, - node, nodelen, - service, servicelen, - flags); - - if (rc != 0) - { - if (node) - strlcpy(node, "???", nodelen); - if (service) - strlcpy(service, "???", servicelen); - } - - return rc; -} - - -#if defined(HAVE_UNIX_SOCKETS) - -/* ------- - * getaddrinfo_unix - get unix socket info using IPv6-compatible API - * - * Bugs: only one addrinfo is set even though hintsp is NULL or - * ai_socktype is 0 - * AI_CANONNAME is not supported. - * ------- - */ -static int -getaddrinfo_unix(const char *path, const struct addrinfo * hintsp, - struct addrinfo ** result) -{ - struct addrinfo hints; - struct addrinfo *aip; - struct sockaddr_un *unp; - - *result = NULL; - - MemSet(&hints, 0, sizeof(hints)); - - if (strlen(path) >= sizeof(unp->sun_path)) - return EAI_FAIL; - - if (hintsp == NULL) - { - hints.ai_family = AF_UNIX; - hints.ai_socktype = SOCK_STREAM; - } - else - memcpy(&hints, hintsp, sizeof(hints)); - - if (hints.ai_socktype == 0) - hints.ai_socktype = SOCK_STREAM; - - if (hints.ai_family != AF_UNIX) - { - /* shouldn't have been called */ - return EAI_FAIL; - } - - aip = calloc(1, sizeof(struct addrinfo)); - if (aip == NULL) - return EAI_MEMORY; - - unp = calloc(1, sizeof(struct sockaddr_un)); - if (unp == NULL) - { - free(aip); - return EAI_MEMORY; - } - - aip->ai_family = AF_UNIX; - aip->ai_socktype = hints.ai_socktype; - aip->ai_protocol = hints.ai_protocol; - aip->ai_next = NULL; - aip->ai_canonname = NULL; - *result = aip; - - unp->sun_family = AF_UNIX; - aip->ai_addr = (struct sockaddr *) unp; - aip->ai_addrlen = sizeof(struct sockaddr_un); - - strcpy(unp->sun_path, path); - -#ifdef HAVE_STRUCT_SOCKADDR_STORAGE_SS_LEN - unp->sun_len = sizeof(struct sockaddr_un); -#endif - - return 0; -} - -/* - * Convert an address to a hostname. - */ -static int -getnameinfo_unix(const struct sockaddr_un * sa, int salen, - char *node, int nodelen, - char *service, int servicelen, - int flags) -{ - int ret = -1; - - /* Invalid arguments. */ - if (sa == NULL || sa->sun_family != AF_UNIX || - (node == NULL && service == NULL)) - return EAI_FAIL; - - if (node) - { - ret = snprintf(node, nodelen, "%s", "[local]"); - if (ret == -1 || ret > nodelen) - return EAI_MEMORY; - } - - if (service) - { - ret = snprintf(service, servicelen, "%s", sa->sun_path); - if (ret == -1 || ret > servicelen) - return EAI_MEMORY; - } - - return 0; -} -#endif /* HAVE_UNIX_SOCKETS */ - /* * pg_range_sockaddr - is addr within the subnet specified by netaddr/netmask ? diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 90b6946b38..bae96bf18f 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -89,7 +89,7 @@ #include #endif -#include "libpq/ip.h" +#include "common/ip.h" #include "libpq/libpq.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 2f99aea791..8a2ce91344 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -38,7 +38,7 @@ #include "access/xact.h" #include "catalog/pg_database.h" #include "catalog/pg_proc.h" -#include "libpq/ip.h" +#include "common/ip.h" #include "libpq/libpq.h" #include "libpq/pqsignal.h" #include "mb/pg_wchar.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index a28e215e2d..eaf3f61ecf 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -99,9 +99,9 @@ #include "access/xlog.h" #include "bootstrap/bootstrap.h" #include "catalog/pg_control.h" +#include "common/ip.h" #include "lib/ilist.h" #include "libpq/auth.h" -#include "libpq/ip.h" #include "libpq/libpq.h" #include "libpq/pqsignal.h" #include "miscadmin.h" diff --git a/src/backend/utils/adt/network.c b/src/backend/utils/adt/network.c index 3f6987af04..dbc557e583 100644 --- a/src/backend/utils/adt/network.c +++ b/src/backend/utils/adt/network.c @@ -14,7 +14,7 @@ #include "access/hash.h" #include "catalog/pg_type.h" -#include "libpq/ip.h" +#include "common/ip.h" #include "libpq/libpq-be.h" #include "libpq/pqformat.h" #include "miscadmin.h" diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 1bba5fa8c8..5d1ccf51d4 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -16,8 +16,8 @@ #include "access/htup_details.h" #include "catalog/pg_type.h" +#include "common/ip.h" #include "funcapi.h" -#include "libpq/ip.h" #include "miscadmin.h" #include "pgstat.h" #include "storage/proc.h" diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index bf7c0cd735..582d3e460b 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -21,8 +21,8 @@ #include "access/tuptoaster.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" +#include "common/md5.h" #include "lib/hyperloglog.h" -#include "libpq/md5.h" #include "libpq/pqformat.h" #include "miscadmin.h" #include "parser/scansup.h" diff --git a/src/common/Makefile b/src/common/Makefile index 72b73697a8..a5fa649766 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -6,11 +6,15 @@ # This makefile generates two outputs: # # libpgcommon.a - contains object files with FRONTEND defined, -# for use by client application and libraries +# for use by client applications # # libpgcommon_srv.a - contains object files without FRONTEND defined, # for use only by the backend binaries # +# You can also symlink/copy individual source files from this directory, +# to compile with different options. (libpq does that, because it needs +# to use -fPIC on some platforms.) +# # IDENTIFICATION # src/common/Makefile # @@ -36,8 +40,8 @@ override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\"" override CPPFLAGS += -DVAL_LDFLAGS_SL="\"$(LDFLAGS_SL)\"" override CPPFLAGS += -DVAL_LIBS="\"$(LIBS)\"" -OBJS_COMMON = config_info.o controldata_utils.o exec.o keywords.o \ - pg_lzcompress.o pgfnames.o psprintf.o relpath.o rmtree.o \ +OBJS_COMMON = config_info.o controldata_utils.o exec.o ip.o keywords.o \ + md5.o pg_lzcompress.o pgfnames.o psprintf.o relpath.o rmtree.o \ string.o username.o wait_error.o OBJS_FRONTEND = $(OBJS_COMMON) fe_memutils.o restricted_token.o diff --git a/src/common/ip.c b/src/common/ip.c new file mode 100644 index 0000000000..797d9109ab --- /dev/null +++ b/src/common/ip.c @@ -0,0 +1,260 @@ +/*------------------------------------------------------------------------- + * + * ip.c + * IPv6-aware network access. + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/common/ip.c + * + * This file and the IPV6 implementation were initially provided by + * Nigel Kukard , Linux Based Systems Design + * http://www.lbsd.net. + * + *------------------------------------------------------------------------- + */ + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include +#include +#include +#include +#include +#include +#ifdef HAVE_NETINET_TCP_H +#include +#endif +#include +#include + +#include "common/ip.h" + + + +#ifdef HAVE_UNIX_SOCKETS +static int getaddrinfo_unix(const char *path, + const struct addrinfo * hintsp, + struct addrinfo ** result); + +static int getnameinfo_unix(const struct sockaddr_un * sa, int salen, + char *node, int nodelen, + char *service, int servicelen, + int flags); +#endif + + +/* + * pg_getaddrinfo_all - get address info for Unix, IPv4 and IPv6 sockets + */ +int +pg_getaddrinfo_all(const char *hostname, const char *servname, + const struct addrinfo * hintp, struct addrinfo ** result) +{ + int rc; + + /* not all versions of getaddrinfo() zero *result on failure */ + *result = NULL; + +#ifdef HAVE_UNIX_SOCKETS + if (hintp->ai_family == AF_UNIX) + return getaddrinfo_unix(servname, hintp, result); +#endif + + /* NULL has special meaning to getaddrinfo(). */ + rc = getaddrinfo((!hostname || hostname[0] == '\0') ? NULL : hostname, + servname, hintp, result); + + return rc; +} + + +/* + * pg_freeaddrinfo_all - free addrinfo structures for IPv4, IPv6, or Unix + * + * Note: the ai_family field of the original hint structure must be passed + * so that we can tell whether the addrinfo struct was built by the system's + * getaddrinfo() routine or our own getaddrinfo_unix() routine. Some versions + * of getaddrinfo() might be willing to return AF_UNIX addresses, so it's + * not safe to look at ai_family in the addrinfo itself. + */ +void +pg_freeaddrinfo_all(int hint_ai_family, struct addrinfo * ai) +{ +#ifdef HAVE_UNIX_SOCKETS + if (hint_ai_family == AF_UNIX) + { + /* struct was built by getaddrinfo_unix (see pg_getaddrinfo_all) */ + while (ai != NULL) + { + struct addrinfo *p = ai; + + ai = ai->ai_next; + free(p->ai_addr); + free(p); + } + } + else +#endif /* HAVE_UNIX_SOCKETS */ + { + /* struct was built by getaddrinfo() */ + if (ai != NULL) + freeaddrinfo(ai); + } +} + + +/* + * pg_getnameinfo_all - get name info for Unix, IPv4 and IPv6 sockets + * + * The API of this routine differs from the standard getnameinfo() definition + * in two ways: first, the addr parameter is declared as sockaddr_storage + * rather than struct sockaddr, and second, the node and service fields are + * guaranteed to be filled with something even on failure return. + */ +int +pg_getnameinfo_all(const struct sockaddr_storage * addr, int salen, + char *node, int nodelen, + char *service, int servicelen, + int flags) +{ + int rc; + +#ifdef HAVE_UNIX_SOCKETS + if (addr && addr->ss_family == AF_UNIX) + rc = getnameinfo_unix((const struct sockaddr_un *) addr, salen, + node, nodelen, + service, servicelen, + flags); + else +#endif + rc = getnameinfo((const struct sockaddr *) addr, salen, + node, nodelen, + service, servicelen, + flags); + + if (rc != 0) + { + if (node) + strlcpy(node, "???", nodelen); + if (service) + strlcpy(service, "???", servicelen); + } + + return rc; +} + + +#if defined(HAVE_UNIX_SOCKETS) + +/* ------- + * getaddrinfo_unix - get unix socket info using IPv6-compatible API + * + * Bugs: only one addrinfo is set even though hintsp is NULL or + * ai_socktype is 0 + * AI_CANONNAME is not supported. + * ------- + */ +static int +getaddrinfo_unix(const char *path, const struct addrinfo * hintsp, + struct addrinfo ** result) +{ + struct addrinfo hints; + struct addrinfo *aip; + struct sockaddr_un *unp; + + *result = NULL; + + MemSet(&hints, 0, sizeof(hints)); + + if (strlen(path) >= sizeof(unp->sun_path)) + return EAI_FAIL; + + if (hintsp == NULL) + { + hints.ai_family = AF_UNIX; + hints.ai_socktype = SOCK_STREAM; + } + else + memcpy(&hints, hintsp, sizeof(hints)); + + if (hints.ai_socktype == 0) + hints.ai_socktype = SOCK_STREAM; + + if (hints.ai_family != AF_UNIX) + { + /* shouldn't have been called */ + return EAI_FAIL; + } + + aip = calloc(1, sizeof(struct addrinfo)); + if (aip == NULL) + return EAI_MEMORY; + + unp = calloc(1, sizeof(struct sockaddr_un)); + if (unp == NULL) + { + free(aip); + return EAI_MEMORY; + } + + aip->ai_family = AF_UNIX; + aip->ai_socktype = hints.ai_socktype; + aip->ai_protocol = hints.ai_protocol; + aip->ai_next = NULL; + aip->ai_canonname = NULL; + *result = aip; + + unp->sun_family = AF_UNIX; + aip->ai_addr = (struct sockaddr *) unp; + aip->ai_addrlen = sizeof(struct sockaddr_un); + + strcpy(unp->sun_path, path); + +#ifdef HAVE_STRUCT_SOCKADDR_STORAGE_SS_LEN + unp->sun_len = sizeof(struct sockaddr_un); +#endif + + return 0; +} + +/* + * Convert an address to a hostname. + */ +static int +getnameinfo_unix(const struct sockaddr_un * sa, int salen, + char *node, int nodelen, + char *service, int servicelen, + int flags) +{ + int ret = -1; + + /* Invalid arguments. */ + if (sa == NULL || sa->sun_family != AF_UNIX || + (node == NULL && service == NULL)) + return EAI_FAIL; + + if (node) + { + ret = snprintf(node, nodelen, "%s", "[local]"); + if (ret == -1 || ret > nodelen) + return EAI_MEMORY; + } + + if (service) + { + ret = snprintf(service, servicelen, "%s", sa->sun_path); + if (ret == -1 || ret > servicelen) + return EAI_MEMORY; + } + + return 0; +} +#endif /* HAVE_UNIX_SOCKETS */ diff --git a/src/backend/libpq/md5.c b/src/common/md5.c similarity index 98% rename from src/backend/libpq/md5.c rename to src/common/md5.c index 5af54e6c56..6dad16529c 100644 --- a/src/backend/libpq/md5.c +++ b/src/common/md5.c @@ -14,13 +14,16 @@ * Portions Copyright (c) 1994, Regents of the University of California * * IDENTIFICATION - * src/backend/libpq/md5.c + * src/common/md5.c */ -/* This is intended to be used in both frontend and backend, so use c.h */ -#include "c.h" +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif -#include "libpq/md5.h" +#include "common/md5.h" /* diff --git a/src/include/libpq/ip.h b/src/include/common/ip.h similarity index 65% rename from src/include/libpq/ip.h rename to src/include/common/ip.h index ce9bc6e225..36e0511324 100644 --- a/src/include/libpq/ip.h +++ b/src/include/common/ip.h @@ -3,12 +3,11 @@ * ip.h * Definitions for IPv6-aware network access. * - * These definitions are used by both frontend and backend code. Be careful - * what you include here! + * These definitions are used by both frontend and backend code. * * Copyright (c) 2003-2016, PostgreSQL Global Development Group * - * src/include/libpq/ip.h + * src/include/common/ip.h * *------------------------------------------------------------------------- */ @@ -25,10 +24,6 @@ #define IS_AF_UNIX(fam) (0) #endif -typedef void (*PgIfAddrCallback) (struct sockaddr * addr, - struct sockaddr * netmask, - void *cb_data); - extern int pg_getaddrinfo_all(const char *hostname, const char *servname, const struct addrinfo * hintp, struct addrinfo ** result); @@ -39,13 +34,4 @@ extern int pg_getnameinfo_all(const struct sockaddr_storage * addr, int salen, char *service, int servicelen, int flags); -extern int pg_range_sockaddr(const struct sockaddr_storage * addr, - const struct sockaddr_storage * netaddr, - const struct sockaddr_storage * netmask); - -extern int pg_sockaddr_cidr_mask(struct sockaddr_storage * mask, - char *numbits, int family); - -extern int pg_foreach_ifaddr(PgIfAddrCallback callback, void *cb_data); - #endif /* IP_H */ diff --git a/src/include/libpq/md5.h b/src/include/common/md5.h similarity index 96% rename from src/include/libpq/md5.h rename to src/include/common/md5.h index f3eec8b4f4..4a0432076a 100644 --- a/src/include/libpq/md5.h +++ b/src/include/common/md5.h @@ -9,7 +9,7 @@ * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * src/include/libpq/md5.h + * src/include/common/md5.h * *------------------------------------------------------------------------- */ diff --git a/src/include/libpq/ifaddr.h b/src/include/libpq/ifaddr.h new file mode 100644 index 0000000000..40094a612a --- /dev/null +++ b/src/include/libpq/ifaddr.h @@ -0,0 +1,30 @@ +/*------------------------------------------------------------------------- + * + * ifaddr.h + * IP netmask calculations, and enumerating network interfaces. + * + * Copyright (c) 2003-2016, PostgreSQL Global Development Group + * + * src/include/libpq/ifaddr.h + * + *------------------------------------------------------------------------- + */ +#ifndef IFADDR_H +#define IFADDR_H + +#include "libpq/pqcomm.h" /* pgrminclude ignore */ + +typedef void (*PgIfAddrCallback) (struct sockaddr * addr, + struct sockaddr * netmask, + void *cb_data); + +extern int pg_range_sockaddr(const struct sockaddr_storage * addr, + const struct sockaddr_storage * netaddr, + const struct sockaddr_storage * netmask); + +extern int pg_sockaddr_cidr_mask(struct sockaddr_storage * mask, + char *numbits, int family); + +extern int pg_foreach_ifaddr(PgIfAddrCallback callback, void *cb_data); + +#endif /* IFADDR_H */ diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 0b4065ed8f..b1789eb35e 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -39,10 +39,10 @@ OBJS += chklocale.o inet_net_ntop.o noblock.o pgstrcasecmp.o pqsignal.o \ thread.o # libpgport C files that are needed if identified by configure OBJS += $(filter crypt.o getaddrinfo.o getpeereid.o inet_aton.o open.o system.o snprintf.o strerror.o strlcpy.o win32error.o win32setlocale.o, $(LIBOBJS)) -# backend/libpq -OBJS += ip.o md5.o -# utils/mb +# src/backend/utils/mb OBJS += encnames.o wchar.o +# src/common +OBJS += ip.o md5.o ifeq ($(with_openssl),yes) OBJS += fe-secure-openssl.o @@ -96,7 +96,7 @@ backend_src = $(top_srcdir)/src/backend chklocale.c crypt.c getaddrinfo.c getpeereid.c inet_aton.c inet_net_ntop.c noblock.c open.c system.c pgsleep.c pgstrcasecmp.c pqsignal.c snprintf.c strerror.c strlcpy.c thread.c win32error.c win32setlocale.c: % : $(top_srcdir)/src/port/% rm -f $@ && $(LN_S) $< . -ip.c md5.c: % : $(backend_src)/libpq/% +ip.c md5.c: % : $(top_srcdir)/src/common/% rm -f $@ && $(LN_S) $< . encnames.c wchar.c: % : $(backend_src)/utils/mb/% diff --git a/src/interfaces/libpq/fe-auth.c b/src/interfaces/libpq/fe-auth.c index d23726215b..404bc93306 100644 --- a/src/interfaces/libpq/fe-auth.c +++ b/src/interfaces/libpq/fe-auth.c @@ -38,9 +38,9 @@ #include #endif +#include "common/md5.h" #include "libpq-fe.h" #include "fe-auth.h" -#include "libpq/md5.h" #ifdef ENABLE_GSS diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 76b61bdc25..9668b52103 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -72,7 +72,7 @@ static int ldapServiceLookup(const char *purl, PQconninfoOption *options, PQExpBuffer errorMessage); #endif -#include "libpq/ip.h" +#include "common/ip.h" #include "mb/pg_wchar.h" #ifndef FD_CLOEXEC diff --git a/src/tools/ifaddrs/Makefile b/src/tools/ifaddrs/Makefile index 231f388901..eed6af41e6 100644 --- a/src/tools/ifaddrs/Makefile +++ b/src/tools/ifaddrs/Makefile @@ -20,8 +20,8 @@ OBJS = test_ifaddrs.o all: test_ifaddrs -test_ifaddrs: test_ifaddrs.o $(libpq_backend_dir)/ip.o - $(CC) $(CFLAGS) test_ifaddrs.o $(libpq_backend_dir)/ip.o $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) +test_ifaddrs: test_ifaddrs.o $(libpq_backend_dir)/ifaddr.o + $(CC) $(CFLAGS) test_ifaddrs.o $(libpq_backend_dir)/ifaddr.o $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) clean distclean maintainer-clean: rm -f test_ifaddrs$(X) $(OBJS) diff --git a/src/tools/ifaddrs/test_ifaddrs.c b/src/tools/ifaddrs/test_ifaddrs.c index 48d184c84a..80b9bb0266 100644 --- a/src/tools/ifaddrs/test_ifaddrs.c +++ b/src/tools/ifaddrs/test_ifaddrs.c @@ -12,7 +12,7 @@ #include #include -#include "libpq/ip.h" +#include "libpq/ifaddr.h" static void diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 6746728616..b3ed1f56e2 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -110,8 +110,8 @@ sub mkvcbuild } our @pgcommonallfiles = qw( - config_info.c controldata_utils.c exec.c keywords.c - pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c + config_info.c controldata_utils.c exec.c ip.c keywords.c + md5.c pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c string.c username.c wait_error.c); our @pgcommonfrontendfiles = ( From 39b691f251167bbb3d49203abfb39d430f68f411 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 2 Sep 2016 17:29:31 -0400 Subject: [PATCH 101/871] Don't require dynamic timezone abbreviations to match underlying time zone. Previously, we threw an error if a dynamic timezone abbreviation did not match any abbreviation recorded in the referenced IANA time zone entry. That seemed like a good consistency check at the time, but it turns out that a number of the abbreviations in the IANA database are things that Olson and crew made up out of whole cloth. Their current policy is to remove such names in favor of using simple numeric offsets. Perhaps unsurprisingly, a lot of these made-up abbreviations have varied in meaning over time, which meant that our commit b2cbced9e and later changes made them into dynamic abbreviations. So with newer IANA database versions that don't mention these abbreviations at all, we fail, as reported in bug #14307 from Neil Anderson. It's worse than just a few unused-in-the-wild abbreviations not working, because the pg_timezone_abbrevs view stops working altogether (since its underlying function tries to compute the whole view result in one call). We considered deleting these abbreviations from our abbreviations list, but the problem with that is that we can't stay ahead of possible future IANA changes. Instead, let's leave the abbreviations list alone, and treat any "orphaned" dynamic abbreviation as just meaning the referenced time zone. It will behave a bit differently than it used to, in that you can't any longer override the zone's standard vs. daylight rule by using the "wrong" abbreviation of a pair, but that's better than failing entirely. (Also, this solution can be interpreted as adding a small new feature, which is that any abbreviation a user wants can be defined as referencing a time zone name.) Back-patch to all supported branches, since this problem affects all of them when using tzdata 2016f or newer. Report: <20160902031551.15674.67337@wrigleys.postgresql.org> Discussion: <6189.1472820913@sss.pgh.pa.us> --- doc/src/sgml/catalogs.sgml | 7 ++ doc/src/sgml/datetime.sgml | 41 ++++++++--- src/backend/utils/adt/datetime.c | 85 +++++++++++++++++------ src/test/regress/expected/timestamptz.out | 20 ++++++ src/test/regress/sql/timestamptz.sql | 11 +++ 5 files changed, 132 insertions(+), 32 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 4e09e06aed..322d8d6dc7 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -9811,6 +9811,13 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx + + While most timezone abbreviations represent fixed offsets from UTC, + there are some that have historically varied in value + (see for more information). + In such cases this view presents their current meaning. + + diff --git a/doc/src/sgml/datetime.sgml b/doc/src/sgml/datetime.sgml index ffd0715128..ef9139f9e3 100644 --- a/doc/src/sgml/datetime.sgml +++ b/doc/src/sgml/datetime.sgml @@ -384,19 +384,38 @@ A zone_abbreviation is just the abbreviation - being defined. The offset is the equivalent - offset in seconds from UTC, positive being east from Greenwich and - negative being west. For example, -18000 would be five hours west - of Greenwich, or North American east coast standard time. D - indicates that the zone name represents local daylight-savings time rather - than standard time. Alternatively, a time_zone_name can - be given, in which case that time zone definition is consulted, and the - abbreviation's meaning in that zone is used. This alternative is - recommended only for abbreviations whose meaning has historically varied, - as looking up the meaning is noticeably more expensive than just using - a fixed integer value. + being defined. An offset is an integer giving + the equivalent offset in seconds from UTC, positive being east from + Greenwich and negative being west. For example, -18000 would be five + hours west of Greenwich, or North American east coast standard time. + D indicates that the zone name represents local + daylight-savings time rather than standard time. + + Alternatively, a time_zone_name can be given, referencing + a zone name defined in the IANA timezone database. The zone's definition + is consulted to see whether the abbreviation is or has been in use in + that zone, and if so, the appropriate meaning is used — that is, + the meaning that was currently in use at the timestamp whose value is + being determined, or the meaning in use immediately before that if it + wasn't current at that time, or the oldest meaning if it was used only + after that time. This behavior is essential for dealing with + abbreviations whose meaning has historically varied. It is also allowed + to define an abbreviation in terms of a zone name in which that + abbreviation does not appear; then using the abbreviation is just + equivalent to writing out the zone name. + + + + + Using a simple integer offset is preferred + when defining an abbreviation whose offset from UTC has never changed, + as such abbreviations are much cheaper to process than those that + require consulting a time zone definition. + + + The @INCLUDE syntax allows inclusion of another file in the .../share/timezonesets/ directory. Inclusion can be nested, diff --git a/src/backend/utils/adt/datetime.c b/src/backend/utils/adt/datetime.c index 965c3b4ff0..45ba7cd906 100644 --- a/src/backend/utils/adt/datetime.c +++ b/src/backend/utils/adt/datetime.c @@ -56,8 +56,9 @@ static void AdjustFractDays(double frac, struct pg_tm * tm, fsec_t *fsec, int scale); static int DetermineTimeZoneOffsetInternal(struct pg_tm * tm, pg_tz *tzp, pg_time_t *tp); -static int DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, const char *abbr, - pg_tz *tzp, int *isdst); +static bool DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, + const char *abbr, pg_tz *tzp, + int *offset, int *isdst); static pg_tz *FetchDynamicTimeZone(TimeZoneAbbrevTable *tbl, const datetkn *tp); @@ -1689,19 +1690,40 @@ DetermineTimeZoneOffsetInternal(struct pg_tm * tm, pg_tz *tzp, pg_time_t *tp) * This differs from the behavior of DetermineTimeZoneOffset() in that a * standard-time or daylight-time abbreviation forces use of the corresponding * GMT offset even when the zone was then in DS or standard time respectively. + * (However, that happens only if we can match the given abbreviation to some + * abbreviation that appears in the IANA timezone data. Otherwise, we fall + * back to doing DetermineTimeZoneOffset().) */ int DetermineTimeZoneAbbrevOffset(struct pg_tm * tm, const char *abbr, pg_tz *tzp) { pg_time_t t; + int zone_offset; + int abbr_offset; + int abbr_isdst; /* * Compute the UTC time we want to probe at. (In event of overflow, we'll * probe at the epoch, which is a bit random but probably doesn't matter.) */ - (void) DetermineTimeZoneOffsetInternal(tm, tzp, &t); + zone_offset = DetermineTimeZoneOffsetInternal(tm, tzp, &t); - return DetermineTimeZoneAbbrevOffsetInternal(t, abbr, tzp, &tm->tm_isdst); + /* + * Try to match the abbreviation to something in the zone definition. + */ + if (DetermineTimeZoneAbbrevOffsetInternal(t, abbr, tzp, + &abbr_offset, &abbr_isdst)) + { + /* Success, so use the abbrev-specific answers. */ + tm->tm_isdst = abbr_isdst; + return abbr_offset; + } + + /* + * No match, so use the answers we already got from + * DetermineTimeZoneOffsetInternal. + */ + return zone_offset; } @@ -1715,19 +1737,41 @@ DetermineTimeZoneAbbrevOffsetTS(TimestampTz ts, const char *abbr, pg_tz *tzp, int *isdst) { pg_time_t t = timestamptz_to_time_t(ts); + int zone_offset; + int abbr_offset; + int tz; + struct pg_tm tm; + fsec_t fsec; - return DetermineTimeZoneAbbrevOffsetInternal(t, abbr, tzp, isdst); + /* + * If the abbrev matches anything in the zone data, this is pretty easy. + */ + if (DetermineTimeZoneAbbrevOffsetInternal(t, abbr, tzp, + &abbr_offset, isdst)) + return abbr_offset; + + /* + * Else, break down the timestamp so we can use DetermineTimeZoneOffset. + */ + if (timestamp2tm(ts, &tz, &tm, &fsec, NULL, tzp) != 0) + ereport(ERROR, + (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), + errmsg("timestamp out of range"))); + + zone_offset = DetermineTimeZoneOffset(&tm, tzp); + *isdst = tm.tm_isdst; + return zone_offset; } /* DetermineTimeZoneAbbrevOffsetInternal() * * Workhorse for above two functions: work from a pg_time_t probe instant. - * DST status is returned into *isdst. + * On success, return GMT offset and DST status into *offset and *isdst. */ -static int -DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, const char *abbr, - pg_tz *tzp, int *isdst) +static bool +DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, const char *abbr, pg_tz *tzp, + int *offset, int *isdst) { char upabbr[TZ_STRLEN_MAX + 1]; unsigned char *p; @@ -1739,18 +1783,17 @@ DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, const char *abbr, *p = pg_toupper(*p); /* Look up the abbrev's meaning at this time in this zone */ - if (!pg_interpret_timezone_abbrev(upabbr, - &t, - &gmtoff, - isdst, - tzp)) - ereport(ERROR, - (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("time zone abbreviation \"%s\" is not used in time zone \"%s\"", - abbr, pg_get_timezone_name(tzp)))); - - /* Change sign to agree with DetermineTimeZoneOffset() */ - return (int) -gmtoff; + if (pg_interpret_timezone_abbrev(upabbr, + &t, + &gmtoff, + isdst, + tzp)) + { + /* Change sign to agree with DetermineTimeZoneOffset() */ + *offset = (int) -gmtoff; + return true; + } + return false; } diff --git a/src/test/regress/expected/timestamptz.out b/src/test/regress/expected/timestamptz.out index 67f26db204..2bfc13ad72 100644 --- a/src/test/regress/expected/timestamptz.out +++ b/src/test/regress/expected/timestamptz.out @@ -2603,3 +2603,23 @@ SELECT '2007-12-09 07:30:00 UTC'::timestamptz AT TIME ZONE 'VET'; Sun Dec 09 03:00:00 2007 (1 row) +-- +-- Test that the pg_timezone_names and pg_timezone_abbrevs views are +-- more-or-less working. We can't test their contents in any great detail +-- without the outputs changing anytime IANA updates the underlying data, +-- but it seems reasonable to expect at least one entry per major meridian. +-- (At the time of writing, the actual counts are around 38 because of +-- zones using fractional GMT offsets, so this is a pretty loose test.) +-- +select count(distinct utc_offset) >= 24 as ok from pg_timezone_names; + ok +---- + t +(1 row) + +select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; + ok +---- + t +(1 row) + diff --git a/src/test/regress/sql/timestamptz.sql b/src/test/regress/sql/timestamptz.sql index c023095bb8..ce9d1c2fa1 100644 --- a/src/test/regress/sql/timestamptz.sql +++ b/src/test/regress/sql/timestamptz.sql @@ -468,3 +468,14 @@ SELECT '2007-12-09 07:00:00 UTC'::timestamptz AT TIME ZONE 'VET'; SELECT '2007-12-09 07:00:01 UTC'::timestamptz AT TIME ZONE 'VET'; SELECT '2007-12-09 07:29:59 UTC'::timestamptz AT TIME ZONE 'VET'; SELECT '2007-12-09 07:30:00 UTC'::timestamptz AT TIME ZONE 'VET'; + +-- +-- Test that the pg_timezone_names and pg_timezone_abbrevs views are +-- more-or-less working. We can't test their contents in any great detail +-- without the outputs changing anytime IANA updates the underlying data, +-- but it seems reasonable to expect at least one entry per major meridian. +-- (At the time of writing, the actual counts are around 38 because of +-- zones using fractional GMT offsets, so this is a pretty loose test.) +-- +select count(distinct utc_offset) >= 24 as ok from pg_timezone_names; +select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; From 0c40ab3a88edf654165e562deee0c303a6ebef5e Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Sat, 3 Sep 2016 16:19:11 +0100 Subject: [PATCH 102/871] Fix wording of logical decoding concepts Be specific about conditions under which we emit >1 copy of message Craig Ringer --- doc/src/sgml/logicaldecoding.sgml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index c42082002e..484915d042 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -12,7 +12,6 @@ Changes are sent out in streams identified by logical replication slots. - Each stream outputs each change exactly once. @@ -204,8 +203,7 @@ $ pg_recvlogical -d postgres --slot test --drop-slot In the context of logical replication, a slot represents a stream of changes that can be replayed to a client in the order they were made on the origin server. Each slot streams a sequence of changes from a single - database, sending each change exactly once (except when peeking forward - in the stream). + database. @@ -221,6 +219,20 @@ $ pg_recvlogical -d postgres --slot test --drop-slot independently of the connection using them and are crash-safe. + + A logical slot will emit each change just once in normal operation. + The current position of each slot is persisted only at checkpoint, so in + the case of a crash the slot may return to an earlier LSN, which will + then cause recent changes to be resent when the server restarts. + Logical decoding clients are responsible for avoiding ill effects from + handling the same message more than once. Clients may wish to record + the last LSN they saw when decoding and skip over any repeated data or + (when using the replication protocol) request that decoding start from + that LSN rather than letting the server determine the start point. + The Replication Progress Tracking feature is designed for this purpose, + refer to replication origins. + + Multiple independent slots may exist for a single database. Each slot has its own state, allowing different consumers to receive changes from From 35250b6ad7a8ece5cfe54c0316c180df19f36c13 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Sat, 3 Sep 2016 17:48:01 +0100 Subject: [PATCH 103/871] New recovery target recovery_target_lsn Michael Paquier --- doc/src/sgml/recovery-config.sgml | 24 ++++++- .../access/transam/recovery.conf.sample | 6 +- src/backend/access/transam/xlog.c | 70 +++++++++++++++++++ src/include/access/xlog.h | 1 + src/test/recovery/t/003_recovery_targets.pl | 28 ++++++-- 5 files changed, 120 insertions(+), 9 deletions(-) diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml index 26af221745..de3fb10f5b 100644 --- a/doc/src/sgml/recovery-config.sgml +++ b/doc/src/sgml/recovery-config.sgml @@ -157,9 +157,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows By default, recovery will recover to the end of the WAL log. The following parameters can be used to specify an earlier stopping point. At most one of recovery_target, - recovery_target_name, recovery_target_time, or - recovery_target_xid can be used; if more than one of these - is specified in the configuration file, the last entry will be used. + recovery_target_lsn, recovery_target_name, + recovery_target_time, or recovery_target_xid + can be used; if more than one of these is specified in the configuration + file, the last entry will be used. @@ -232,6 +233,23 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows + + + recovery_target_lsn (pg_lsn) + + recovery_target_lsn recovery parameter + + + + + This parameter specifies the LSN of the transaction log location up + to which recovery will proceed. The precise stopping point is also + influenced by . This + parameter is parsed using the system data type + pg_lsn. + + + diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample index b777400d03..7a16751541 100644 --- a/src/backend/access/transam/recovery.conf.sample +++ b/src/backend/access/transam/recovery.conf.sample @@ -67,8 +67,8 @@ # must set a recovery target. # # You may set a recovery target either by transactionId, by name, -# or by timestamp. Recovery may either include or exclude the -# transaction(s) with the recovery target value (ie, stop either +# by timestamp or by WAL position (LSN). Recovery may either include or +# exclude the transaction(s) with the recovery target value (ie, stop either # just after or just before the given target, respectively). # # @@ -78,6 +78,8 @@ # #recovery_target_xid = '' # +#recovery_target_lsn = '' # e.g. '0/70006B8' +# #recovery_target_inclusive = true # # diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 0b991bb91d..2189c22c64 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -67,6 +67,7 @@ #include "utils/builtins.h" #include "utils/guc.h" #include "utils/memutils.h" +#include "utils/pg_lsn.h" #include "utils/ps_status.h" #include "utils/relmapper.h" #include "utils/snapmgr.h" @@ -254,6 +255,7 @@ static RecoveryTargetAction recoveryTargetAction = RECOVERY_TARGET_ACTION_PAUSE; static TransactionId recoveryTargetXid; static TimestampTz recoveryTargetTime; static char *recoveryTargetName; +static XLogRecPtr recoveryTargetLSN; static int recovery_min_apply_delay = 0; static TimestampTz recoveryDelayUntilTime; @@ -275,6 +277,7 @@ static bool fast_promote = false; */ static TransactionId recoveryStopXid; static TimestampTz recoveryStopTime; +static XLogRecPtr recoveryStopLSN; static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; @@ -5078,6 +5081,23 @@ readRecoveryCommandFile(void) (errmsg_internal("recovery_target_name = '%s'", recoveryTargetName))); } + else if (strcmp(item->name, "recovery_target_lsn") == 0) + { + recoveryTarget = RECOVERY_TARGET_LSN; + + /* + * Convert the LSN string given by the user to XLogRecPtr form. + */ + recoveryTargetLSN = + DatumGetLSN(DirectFunctionCall3(pg_lsn_in, + CStringGetDatum(item->value), + ObjectIdGetDatum(InvalidOid), + Int32GetDatum(-1))); + ereport(DEBUG2, + (errmsg_internal("recovery_target_lsn = '%X/%X'", + (uint32) (recoveryTargetLSN >> 32), + (uint32) recoveryTargetLSN))); + } else if (strcmp(item->name, "recovery_target") == 0) { if (strcmp(item->value, "immediate") == 0) @@ -5400,8 +5420,26 @@ recoveryStopsBefore(XLogReaderState *record) recoveryStopAfter = false; recoveryStopXid = InvalidTransactionId; + recoveryStopLSN = InvalidXLogRecPtr; + recoveryStopTime = 0; + recoveryStopName[0] = '\0'; + return true; + } + + /* Check if target LSN has been reached */ + if (recoveryTarget == RECOVERY_TARGET_LSN && + !recoveryTargetInclusive && + record->ReadRecPtr >= recoveryTargetLSN) + { + recoveryStopAfter = false; + recoveryStopXid = InvalidTransactionId; + recoveryStopLSN = record->ReadRecPtr; recoveryStopTime = 0; recoveryStopName[0] = '\0'; + ereport(LOG, + (errmsg("recovery stopping before WAL position (LSN) \"%X/%X\"", + (uint32) (recoveryStopLSN >> 32), + (uint32) recoveryStopLSN))); return true; } @@ -5479,6 +5517,7 @@ recoveryStopsBefore(XLogReaderState *record) recoveryStopAfter = false; recoveryStopXid = recordXid; recoveryStopTime = recordXtime; + recoveryStopLSN = InvalidXLogRecPtr; recoveryStopName[0] = '\0'; if (isCommit) @@ -5532,6 +5571,7 @@ recoveryStopsAfter(XLogReaderState *record) { recoveryStopAfter = true; recoveryStopXid = InvalidTransactionId; + recoveryStopLSN = InvalidXLogRecPtr; (void) getRecordTimestamp(record, &recoveryStopTime); strlcpy(recoveryStopName, recordRestorePointData->rp_name, MAXFNAMELEN); @@ -5543,6 +5583,23 @@ recoveryStopsAfter(XLogReaderState *record) } } + /* Check if the target LSN has been reached */ + if (recoveryTarget == RECOVERY_TARGET_LSN && + recoveryTargetInclusive && + record->ReadRecPtr >= recoveryTargetLSN) + { + recoveryStopAfter = true; + recoveryStopXid = InvalidTransactionId; + recoveryStopLSN = record->ReadRecPtr; + recoveryStopTime = 0; + recoveryStopName[0] = '\0'; + ereport(LOG, + (errmsg("recovery stopping after WAL position (LSN) \"%X/%X\"", + (uint32) (recoveryStopLSN >> 32), + (uint32) recoveryStopLSN))); + return true; + } + if (rmid != RM_XACT_ID) return false; @@ -5598,6 +5655,7 @@ recoveryStopsAfter(XLogReaderState *record) recoveryStopAfter = true; recoveryStopXid = recordXid; recoveryStopTime = recordXtime; + recoveryStopLSN = InvalidXLogRecPtr; recoveryStopName[0] = '\0'; if (xact_info == XLOG_XACT_COMMIT || @@ -5629,6 +5687,7 @@ recoveryStopsAfter(XLogReaderState *record) recoveryStopAfter = true; recoveryStopXid = InvalidTransactionId; recoveryStopTime = 0; + recoveryStopLSN = InvalidXLogRecPtr; recoveryStopName[0] = '\0'; return true; } @@ -6055,6 +6114,11 @@ StartupXLOG(void) ereport(LOG, (errmsg("starting point-in-time recovery to \"%s\"", recoveryTargetName))); + else if (recoveryTarget == RECOVERY_TARGET_LSN) + ereport(LOG, + (errmsg("starting point-in-time recovery to WAL position (LSN) \"%X/%X\"", + (uint32) (recoveryTargetLSN >> 32), + (uint32) recoveryTargetLSN))); else if (recoveryTarget == RECOVERY_TARGET_IMMEDIATE) ereport(LOG, (errmsg("starting point-in-time recovery to earliest consistent point"))); @@ -7124,6 +7188,12 @@ StartupXLOG(void) "%s %s\n", recoveryStopAfter ? "after" : "before", timestamptz_to_str(recoveryStopTime)); + else if (recoveryTarget == RECOVERY_TARGET_LSN) + snprintf(reason, sizeof(reason), + "%s LSN %X/%X\n", + recoveryStopAfter ? "after" : "before", + (uint32 ) (recoveryStopLSN >> 32), + (uint32) recoveryStopLSN); else if (recoveryTarget == RECOVERY_TARGET_NAME) snprintf(reason, sizeof(reason), "at restore point \"%s\"", diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 14b7f7f459..c9f332c908 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -83,6 +83,7 @@ typedef enum RECOVERY_TARGET_XID, RECOVERY_TARGET_TIME, RECOVERY_TARGET_NAME, + RECOVERY_TARGET_LSN, RECOVERY_TARGET_IMMEDIATE } RecoveryTargetType; diff --git a/src/test/recovery/t/003_recovery_targets.pl b/src/test/recovery/t/003_recovery_targets.pl index d1f6d78388..a82545bf6f 100644 --- a/src/test/recovery/t/003_recovery_targets.pl +++ b/src/test/recovery/t/003_recovery_targets.pl @@ -3,7 +3,7 @@ use warnings; use PostgresNode; use TestLib; -use Test::More tests => 7; +use Test::More tests => 9; # Create and test a standby from given backup, with a certain # recovery target. @@ -86,6 +86,16 @@ sub test_recovery_standby $node_master->safe_psql('postgres', "SELECT pg_create_restore_point('$recovery_name');"); +# And now for a recovery target LSN +$node_master->safe_psql('postgres', + "INSERT INTO tab_int VALUES (generate_series(4001,5000))"); +my $recovery_lsn = $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location()"); +my $lsn5 = + $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();"); + +$node_master->safe_psql('postgres', + "INSERT INTO tab_int VALUES (generate_series(5001,6000))"); + # Force archiving of WAL file $node_master->safe_psql('postgres', "SELECT pg_switch_xlog()"); @@ -102,6 +112,9 @@ sub test_recovery_standby @recovery_params = ("recovery_target_name = '$recovery_name'"); test_recovery_standby('name', 'standby_4', $node_master, \@recovery_params, "4000", $lsn4); +@recovery_params = ("recovery_target_lsn = '$recovery_lsn'"); +test_recovery_standby('LSN', 'standby_5', $node_master, \@recovery_params, + "5000", $lsn5); # Multiple targets # Last entry has priority (note that an array respects the order of items @@ -111,16 +124,23 @@ sub test_recovery_standby "recovery_target_xid = '$recovery_txid'", "recovery_target_time = '$recovery_time'"); test_recovery_standby('name + XID + time', - 'standby_5', $node_master, \@recovery_params, "3000", $lsn3); + 'standby_6', $node_master, \@recovery_params, "3000", $lsn3); @recovery_params = ( "recovery_target_time = '$recovery_time'", "recovery_target_name = '$recovery_name'", "recovery_target_xid = '$recovery_txid'"); test_recovery_standby('time + name + XID', - 'standby_6', $node_master, \@recovery_params, "2000", $lsn2); + 'standby_7', $node_master, \@recovery_params, "2000", $lsn2); @recovery_params = ( "recovery_target_xid = '$recovery_txid'", "recovery_target_time = '$recovery_time'", "recovery_target_name = '$recovery_name'"); test_recovery_standby('XID + time + name', - 'standby_7', $node_master, \@recovery_params, "4000", $lsn4); + 'standby_8', $node_master, \@recovery_params, "4000", $lsn4); +@recovery_params = ( + "recovery_target_xid = '$recovery_txid'", + "recovery_target_time = '$recovery_time'", + "recovery_target_name = '$recovery_name'", + "recovery_target_lsn = '$recovery_lsn'",); +test_recovery_standby('XID + time + name + LSN', + 'standby_9', $node_master, \@recovery_params, "5000", $lsn5); From 60893786d5180f5dd5aefd44d9cb6955d77b0473 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 3 Sep 2016 13:28:53 -0400 Subject: [PATCH 104/871] Fix corrupt GIN_SEGMENT_ADDITEMS WAL records on big-endian hardware. computeLeafRecompressWALData() tried to produce a uint16 WAL log field by memcpy'ing the first two bytes of an int-sized variable. That accidentally works on little-endian hardware, but not at all on big-endian. Replay then thinks it's looking at an ADDITEMS action with zero entries, and reads the first two bytes of the first TID therein as the next segno/action, typically leading to "unexpected GIN leaf action" errors during replay. Even if replay failed to crash, the resulting GIN index page would surely be incorrect. To fix, just declare the variable as uint16 instead. Per bug #14295 from Spencer Thomason (much thanks to Spencer for turning his problem into a self-contained test case). This likely also explains a previous report of the same symptom from Bernd Helmle. Back-patch to 9.4 where the problem was introduced (by commit 14d02f0bb). Discussion: <20160826072658.15676.7628@wrigleys.postgresql.org> Possible-Report: <2DA7350F7296B2A142272901@eje.land.credativ.lan> --- src/backend/access/gin/gindatapage.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/access/gin/gindatapage.c b/src/backend/access/gin/gindatapage.c index 97c8bf78e7..276376a6f1 100644 --- a/src/backend/access/gin/gindatapage.c +++ b/src/backend/access/gin/gindatapage.c @@ -86,7 +86,7 @@ typedef struct char action; ItemPointerData *modifieditems; - int nmodifieditems; + uint16 nmodifieditems; /* * The following fields represent the items in this segment. If 'items' is From 600dc4c0da3b8c094ccc1ae75b47c8320898c714 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 3 Sep 2016 14:18:55 -0400 Subject: [PATCH 105/871] Fix multiple bugs in numeric_poly_deserialize(). These were evidently introduced by yesterday's commit 9cca11c91, which perhaps needs more review than it got. Per report from Andreas Seltenreich and additional examination of nearby code. Report: <87oa45qfwq.fsf@credativ.de> --- src/backend/utils/adt/numeric.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index 27efd310ab..384e672c83 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -4193,11 +4193,11 @@ numeric_poly_deserialize(PG_FUNCTION_ARGS) accum_sum_add(&result->sumX, &sumX_var); #endif - set_var_from_num(DatumGetNumeric(sumX2), &sumX2_var); + init_var_from_num(DatumGetNumeric(sumX2), &sumX2_var); #ifdef HAVE_INT128 numericvar_to_int128(&sumX2_var, &result->sumX2); #else - accum_sum_add(&result->sumX2, &sumX_var); + accum_sum_add(&result->sumX2, &sumX2_var); #endif pq_getmsgend(&buf); From 6591f4226c81104f7746da6a5c00519919c560ae Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 3 Sep 2016 15:29:03 -0400 Subject: [PATCH 106/871] Improve readability of the output of psql's \timing command. In addition to the existing decimal-milliseconds output value, display the same value in mm:ss.fff format if it exceeds one second. Tack on hours and even days fields if the interval is large enough. This avoids needing mental arithmetic to convert the values into customary time units. Corey Huinker, reviewed by Gerdan Santos; bikeshedding by many Discussion: --- doc/src/sgml/ref/psql-ref.sgml | 7 +++-- src/bin/psql/common.c | 56 ++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index 8a66ce7983..4806e77be7 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -2789,8 +2789,11 @@ testdb=> \setenv LESS -imx4F \timing [ on | off ] - Without parameter, toggles a display of how long each SQL statement - takes, in milliseconds. With parameter, sets same. + With a parameter, turns displaying of how long each SQL statement + takes on or off. Without a parameter, toggles the display between + on and off. The display is in milliseconds; intervals longer than + 1 second are also shown in minutes:seconds format, with hours and + days fields added if needed. diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c index 7399950284..a7789dfa53 100644 --- a/src/bin/psql/common.c +++ b/src/bin/psql/common.c @@ -10,6 +10,7 @@ #include #include +#include #include #ifndef WIN32 #include /* for write() */ @@ -531,6 +532,57 @@ ClearOrSaveResult(PGresult *result) } +/* + * Print microtiming output. Always print raw milliseconds; if the interval + * is >= 1 second, also break it down into days/hours/minutes/seconds. + */ +static void +PrintTiming(double elapsed_msec) +{ + double seconds; + double minutes; + double hours; + double days; + + if (elapsed_msec < 1000.0) + { + /* This is the traditional (pre-v10) output format */ + printf(_("Time: %.3f ms\n"), elapsed_msec); + return; + } + + /* + * Note: we could print just seconds, in a format like %06.3f, when the + * total is less than 1min. But that's hard to interpret unless we tack + * on "s" or otherwise annotate it. Forcing the display to include + * minutes seems like a better solution. + */ + seconds = elapsed_msec / 1000.0; + minutes = floor(seconds / 60.0); + seconds -= 60.0 * minutes; + if (minutes < 60.0) + { + printf(_("Time: %.3f ms (%02d:%06.3f)\n"), + elapsed_msec, (int) minutes, seconds); + return; + } + + hours = floor(minutes / 60.0); + minutes -= 60.0 * hours; + if (hours < 24.0) + { + printf(_("Time: %.3f ms (%02d:%02d:%06.3f)\n"), + elapsed_msec, (int) hours, (int) minutes, seconds); + return; + } + + days = floor(hours / 24.0); + hours -= 24.0 * days; + printf(_("Time: %.3f ms (%.0f d %02d:%02d:%06.3f)\n"), + elapsed_msec, days, (int) hours, (int) minutes, seconds); +} + + /* * PSQLexec * @@ -679,7 +731,7 @@ PSQLexecWatch(const char *query, const printQueryOpt *opt) /* Possible microtiming output */ if (pset.timing) - printf(_("Time: %.3f ms\n"), elapsed_msec); + PrintTiming(elapsed_msec); return 1; } @@ -1332,7 +1384,7 @@ SendQuery(const char *query) /* Possible microtiming output */ if (pset.timing) - printf(_("Time: %.3f ms\n"), elapsed_msec); + PrintTiming(elapsed_msec); /* check for events that may occur during query execution */ From e21db14b8a6696a2b704b89df9c4be9cd0ea8a33 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sun, 4 Sep 2016 15:02:06 +0300 Subject: [PATCH 107/871] Clarify the new Red-Black post-order traversal code a bit. Coverity complained about the for(;;) loop, because it never actually iterated. It was used just to be able to use "break" to exit it early. I agree with Coverity, that's a bit confusing, so refactor the code to use if-else instead. While we're at it, use a local variable to hold the "current" node. That's shorter and clearer than referring to "iter->last_visited" all the time. --- src/backend/lib/rbtree.c | 46 +++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/backend/lib/rbtree.c b/src/backend/lib/rbtree.c index 242e9803cc..7fb7e55f71 100644 --- a/src/backend/lib/rbtree.c +++ b/src/backend/lib/rbtree.c @@ -781,27 +781,30 @@ static RBNode * rb_inverted_iterator(RBTreeIterator *iter) { RBNode *came_from; + RBNode *current; + + current = iter->last_visited; loop: switch ((InvertedWalkNextStep) iter->next_step) { /* First call, begin from root */ case NextStepBegin: - iter->last_visited = iter->rb->root; + current = iter->rb->root; iter->next_step = NextStepLeft; goto loop; case NextStepLeft: - while (iter->last_visited->left != RBNIL) - iter->last_visited = iter->last_visited->left; + while (current->left != RBNIL) + current = current->left; iter->next_step = NextStepRight; goto loop; case NextStepRight: - if (iter->last_visited->right != RBNIL) + if (current->right != RBNIL) { - iter->last_visited = iter->last_visited->right; + current = current->right; iter->next_step = NextStepLeft; goto loop; } @@ -810,30 +813,29 @@ rb_inverted_iterator(RBTreeIterator *iter) break; case NextStepUp: - for (;;) + came_from = current; + current = current->parent; + if (current == NULL) + { + iter->is_over = true; + break; /* end of iteration */ + } + else if (came_from == current->right) + { + /* return current, then continue to go up */ + break; + } + else { - came_from = iter->last_visited; - iter->last_visited = iter->last_visited->parent; - if (iter->last_visited == NULL) - { - iter->is_over = true; - break; /* end of iteration */ - } - - if (came_from == iter->last_visited->right) - { - /* return current, then continue to go up */ - break; - } - /* otherwise we came from the left */ + Assert(came_from == current->left); iter->next_step = NextStepRight; goto loop; } - break; } - return iter->last_visited; + iter->last_visited = current; + return current; } /* From a2d75b67bccd24d17e328360080e4877d23bc369 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 4 Sep 2016 12:33:58 -0400 Subject: [PATCH 108/871] Remove useless pg_strdup() operations. split_to_stringlist() doesn't modify its first argument nor expect it to remain valid after exit, so there's no need to duplicate the optarg string at the call sites. Per Coverity. (This has been wrong all along, but commit 052cc223d changed the useless calls from "strdup" to "pg_strdup", which apparently made Coverity think it's a new bug. It's not, but it's also not worth back-patching.) --- src/test/regress/pg_regress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 2260057840..14c87c91ab 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -2064,7 +2064,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc * before we add the specified one. */ free_stringlist(&dblist); - split_to_stringlist(pg_strdup(optarg), ", ", &dblist); + split_to_stringlist(optarg, ", ", &dblist); break; case 2: debug = true; @@ -2114,7 +2114,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc dlpath = pg_strdup(optarg); break; case 18: - split_to_stringlist(pg_strdup(optarg), ", ", &extraroles); + split_to_stringlist(optarg, ", ", &extraroles); break; case 19: add_stringlist_item(&temp_configs, optarg); From 5a072244919a92b2c757b2e3985191f02d674627 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 4 Sep 2016 13:19:54 -0400 Subject: [PATCH 109/871] Update release notes to mention need for ALTER EXTENSION UPDATE. Maybe we ought to make pg_upgrade do this for you, but it won't happen in 9.6, so call out the need for it as a migration consideration. --- doc/src/sgml/release-9.6.sgml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 895d88e768..578c3d1fdb 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -303,6 +303,26 @@ This commit is also listed under libpq and psql + + + + Update extension functions to be marked parallel-safe where + appropriate (Andreas Karlsson) + + + + Many of the standard extensions have been updated to allow their + functions to be executed within parallel query worker processes. + These changes will not take effect in + databases pg_upgrade'd from prior versions unless + you apply ALTER EXTENSION UPDATE to each such extension + (in each database of a cluster). + + + From da6ea70c32bed99ca040a5e04d83c0edd5dfc615 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 4 Sep 2016 19:42:08 -0400 Subject: [PATCH 110/871] Remove vestigial references to "zic" in favor of "IANA database". Commit b2cbced9e instituted a policy of referring to the timezone database as the "IANA timezone database" in our user-facing documentation. Propagate that wording into a couple of places that were still using "zic" to refer to the database, which is definitely not right (zic is the compilation tool, not the data). Back-patch, not because this is very important in itself, but because we routinely cherry-pick updates to the tznames files and I don't want to risk future merge failures. --- src/timezone/tznames/Africa.txt | 2 +- src/timezone/tznames/America.txt | 12 +++--- src/timezone/tznames/Asia.txt | 12 +++--- src/timezone/tznames/Australia | 12 +++--- src/timezone/tznames/Australia.txt | 36 ++++++++--------- src/timezone/tznames/Default | 62 +++++++++++++++--------------- src/timezone/tznames/Etc.txt | 2 +- src/timezone/tznames/Europe.txt | 8 ++-- src/timezone/tznames/Pacific.txt | 12 +++--- src/tools/RELEASE_CHANGES | 2 +- 10 files changed, 80 insertions(+), 80 deletions(-) diff --git a/src/timezone/tznames/Africa.txt b/src/timezone/tznames/Africa.txt index f80676dc33..0bd0c405f6 100644 --- a/src/timezone/tznames/Africa.txt +++ b/src/timezone/tznames/Africa.txt @@ -144,7 +144,7 @@ GMT 0 # Greenwich Mean Time # (Europe/London) # CONFLICT! SAST is not unique # Other timezones: -# - SAST South Australian Standard Time (not in zic) +# - SAST South Australian Standard Time (not in IANA database) SAST 7200 # South Africa Standard Time # (Africa/Johannesburg) WAST 7200 D # West Africa Summer Time diff --git a/src/timezone/tznames/America.txt b/src/timezone/tznames/America.txt index b76c311f50..f00834e987 100644 --- a/src/timezone/tznames/America.txt +++ b/src/timezone/tznames/America.txt @@ -15,7 +15,7 @@ ACT -18000 # Acre Time # CONFLICT! ACST is not unique # Other timezones: # - ACST: Australian Central Standard Time -ACST -14400 D # Acre Summer Time (obsolete, not in zic) +ACST -14400 D # Acre Summer Time (obsolete, not in IANA database) ADT -10800 D # Atlantic Daylight Time # (America/Glace_Bay) # (America/Goose_Bay) @@ -92,7 +92,7 @@ AST -14400 # Atlantic Standard Time # (Atlantic/Bermuda) BOT -14400 # Bolivia Time # (America/La_Paz) -BRA -10800 # Brazil Time (not in zic) +BRA -10800 # Brazil Time (not in IANA database) BRST -7200 D # Brasil Summer Time # (America/Sao_Paulo) BRT -10800 # Brasil Time @@ -131,7 +131,7 @@ CLST -10800 D # Chile Summer Time CLT America/Santiago # Chile Time # (America/Santiago) # (Antarctica/Palmer) -COT -18000 # Columbia Time (not in zic) +COT -18000 # Columbia Time (not in IANA database) # CONFLICT! CST is not unique # Other timezones: # - CST: Central Standard Time (Australia) @@ -207,7 +207,7 @@ EST -18000 # Eastern Standard Time (America) # (America/Toronto) FNT -7200 # Fernando de Noronha Time # (America/Noronha) -FNST -3600 D # Fernando de Noronha Summer Time (not in zic) +FNST -3600 D # Fernando de Noronha Summer Time (not in IANA database) # (America/Noronha) GFT -10800 # French Guiana Time # (America/Cayenne) @@ -265,7 +265,7 @@ NDT -9000 D # Newfoundland Daylight Time # CONFLICT! NFT is not unique # Other timezones: # - NFT: Norfolk Time (Pacific) -NFT -12600 # Newfoundland Time (not in zic) +NFT -12600 # Newfoundland Time (not in IANA database) NST -12600 # Newfoundland Standard Time # (America/St_Johns) PDT -25200 D # Pacific Daylight Time @@ -274,7 +274,7 @@ PDT -25200 D # Pacific Daylight Time # (America/Tijuana) # (America/Vancouver) # (America/Whitehorse) -PET -18000 # Peru Time (not in zic) +PET -18000 # Peru Time (not in IANA database) PMDT -7200 D # Pierre & Miquelon Daylight Time # (America/Miquelon) PMST -10800 # Pierre & Miquelon Standard Time diff --git a/src/timezone/tznames/Asia.txt b/src/timezone/tznames/Asia.txt index 42244501b8..f17dbfe9f2 100644 --- a/src/timezone/tznames/Asia.txt +++ b/src/timezone/tznames/Asia.txt @@ -48,10 +48,10 @@ BDT 21600 # Bangladesh Time # (Asia/Dhaka) BNT 28800 # Brunei Darussalam Time # (Asia/Brunei) -BORT 28800 # Borneo Time (Indonesia) (not in zic) +BORT 28800 # Borneo Time (Indonesia) (not in IANA database) BTT 21600 # Bhutan Time # (Asia/Thimphu) -CCT 28800 # China Coastal Time (not in zic) +CCT 28800 # China Coastal Time (not in IANA database) CHOST Asia/Choibalsan # Choibalsan Summer Time # (Asia/Choibalsan) CHOT Asia/Choibalsan # Choibalsan Time @@ -123,7 +123,7 @@ GET Asia/Tbilisi # Georgia Time GST 14400 # Gulf Standard Time # (Asia/Dubai) # (Asia/Muscat) -HKT 28800 # Hong Kong Time (not in zic) +HKT 28800 # Hong Kong Time (not in IANA database) HOVST 28800 D # Hovd Summer Time # (Asia/Hovd) HOVT Asia/Hovd # Hovd Time @@ -142,7 +142,7 @@ IRKT Asia/Irkutsk # Irkutsk Time # (Asia/Irkutsk) IRST Asia/Tehran # Iran Standard Time # (Asia/Tehran) -IRT 12600 # Iran Time (not in zic) +IRT 12600 # Iran Time (not in IANA database) # CONFLICT! IST is not unique # Other timezones: # - IST: Irish Summer Time (Europe) @@ -155,10 +155,10 @@ IST 19800 # Indian Standard Time # - IST: Indian Standard Time (Asia) IST 7200 # Israel Standard Time # (Asia/Jerusalem) -JAYT 32400 # Jayapura Time (Indonesia) (not in zic) +JAYT 32400 # Jayapura Time (Indonesia) (not in IANA database) JST 32400 # Japan Standard Time # (Asia/Tokyo) -KDT 36000 D # Korean Daylight Time (not in zic) +KDT 36000 D # Korean Daylight Time (not in IANA database) KGST 21600 D # Kyrgyzstan Summer Time (obsolete) KGT Asia/Bishkek # Kyrgyzstan Time # (Asia/Bishkek) diff --git a/src/timezone/tznames/Australia b/src/timezone/tznames/Australia index f888fbcd6c..7216e06d6a 100644 --- a/src/timezone/tznames/Australia +++ b/src/timezone/tznames/Australia @@ -19,9 +19,9 @@ # in case of a conflict. @OVERRIDE -CST 34200 # Central Standard Time (not in zic) -EAST 36000 # East Australian Standard Time (not in zic) -EST 36000 # Eastern Standard Time (not in zic) -SAST 34200 # South Australian Standard Time (not in zic) -SAT 34200 # South Australian Standard Time (not in zic) -WST 28800 # Western Standard Time (not in zic) +CST 34200 # Central Standard Time (not in IANA database) +EAST 36000 # East Australian Standard Time (not in IANA database) +EST 36000 # Eastern Standard Time (not in IANA database) +SAST 34200 # South Australian Standard Time (not in IANA database) +SAT 34200 # South Australian Standard Time (not in IANA database) +WST 28800 # Western Standard Time (not in IANA database) diff --git a/src/timezone/tznames/Australia.txt b/src/timezone/tznames/Australia.txt index 8f1c1379c1..9751c3deb1 100644 --- a/src/timezone/tznames/Australia.txt +++ b/src/timezone/tznames/Australia.txt @@ -7,7 +7,7 @@ # src/timezone/tznames/Australia.txt # -ACSST 37800 D # Central Australia (not in zic) +ACSST 37800 D # Central Australia (not in IANA database) ACDT 37800 D # Australian Central Daylight Time # (Australia/Adelaide) # (Australia/Broken_Hill) @@ -18,7 +18,7 @@ ACST 34200 # Australian Central Standard Time # (Australia/Darwin) ACWST 31500 # Australian Central Western Standard Time # (Australia/Eucla) -AESST 39600 D # Australia Eastern Summer Standard Time (not in zic) +AESST 39600 D # Australia Eastern Summer Standard Time (not in IANA database) AEDT 39600 D # Australian Eastern Daylight Time # (Australia/Brisbane) # (Australia/Currie) @@ -33,42 +33,42 @@ AEST 36000 # Australian Eastern Standard Time # (Australia/Lindeman) # (Australia/Melbourne) # (Australia/Sydney) -AWSST 32400 D # Australia Western Summer Standard Time (not in zic) +AWSST 32400 D # Australia Western Summer Standard Time (not in IANA database) AWST 28800 # Australian Western Standard Time # (Australia/Perth) -CADT 37800 D # Central Australia Daylight-Saving Time (not in zic) -CAST 34200 # Central Australia Standard Time (not in zic) +CADT 37800 D # Central Australia Daylight-Saving Time (not in IANA database) +CAST 34200 # Central Australia Standard Time (not in IANA database) # CONFLICT! CST is not unique # Other timezones: # - CST: Central Standard Time (America) # - CST: China Standard Time (Asia) # - CST: Cuba Central Standard Time (America) -CST 34200 # Central Standard Time (not in zic) -CWST 31500 # Central Western Standard Time (not in zic) +CST 34200 # Central Standard Time (not in IANA database) +CWST 31500 # Central Western Standard Time (not in IANA database) # CONFLICT! EAST is not unique # Other timezones: # - EAST: Easter Island Time (Chile) (Pacific) -EAST 36000 # East Australian Standard Time (not in zic) +EAST 36000 # East Australian Standard Time (not in IANA database) # CONFLICT! EST is not unique # Other timezones: # - EST: Eastern Standard Time (America) -EST 36000 # Eastern Standard Time (not in zic) +EST 36000 # Eastern Standard Time (not in IANA database) LHDT Australia/Lord_Howe # Lord Howe Daylight Time # (Australia/Lord_Howe) LHST 37800 # Lord Howe Standard Time # (Australia/Lord_Howe) -LIGT 36000 # Melbourne, Australia (not in zic) -NZT 43200 # New Zealand Time (not in zic) -SADT 37800 D # South Australian Daylight-Saving Time (not in zic) +LIGT 36000 # Melbourne, Australia (not in IANA database) +NZT 43200 # New Zealand Time (not in IANA database) +SADT 37800 D # South Australian Daylight-Saving Time (not in IANA database) # CONFLICT! SAST is not unique # Other timezones: # - SAST South Africa Standard Time -SAST 34200 # South Australian Standard Time (not in zic) -SAT 34200 # South Australian Standard Time (not in zic) -WADT 28800 D # West Australian Daylight-Saving Time (not in zic) -WAST 25200 # West Australian Standard Time (not in zic) -WDT 32400 D # West Australian Daylight-Saving Time (not in zic) +SAST 34200 # South Australian Standard Time (not in IANA database) +SAT 34200 # South Australian Standard Time (not in IANA database) +WADT 28800 D # West Australian Daylight-Saving Time (not in IANA database) +WAST 25200 # West Australian Standard Time (not in IANA database) +WDT 32400 D # West Australian Daylight-Saving Time (not in IANA database) # CONFLICT! WST is not unique # Other timezones: # - WST: West Samoa Time -WST 28800 # Western Standard Time (not in zic) +WST 28800 # Western Standard Time (not in IANA database) diff --git a/src/timezone/tznames/Default b/src/timezone/tznames/Default index fe82a2fac3..7644e264b0 100644 --- a/src/timezone/tznames/Default +++ b/src/timezone/tznames/Default @@ -68,7 +68,7 @@ ART America/Argentina/Buenos_Aires # Argentina Time ARST America/Argentina/Buenos_Aires # Argentina Summer Time BOT -14400 # Bolivia Time # (America/La_Paz) -BRA -10800 # Brazil Time (not in zic) +BRA -10800 # Brazil Time (not in IANA database) BRST -7200 D # Brasil Summer Time # (America/Sao_Paulo) BRT -10800 # Brasil Time @@ -79,7 +79,7 @@ BRT -10800 # Brasil Time # (America/Maceio) # (America/Recife) # (America/Sao_Paulo) -COT -18000 # Columbia Time (not in zic) +COT -18000 # Columbia Time (not in IANA database) # CONFLICT! CDT is not unique # Other timezones: # - CDT: Mexico Central Daylight Time (America) @@ -166,7 +166,7 @@ EST -18000 # Eastern Standard Time (America) # (America/Toronto) FNT -7200 # Fernando de Noronha Time # (America/Noronha) -FNST -3600 D # Fernando de Noronha Summer Time (not in zic) +FNST -3600 D # Fernando de Noronha Summer Time (not in IANA database) # (America/Noronha) GFT -10800 # French Guiana Time # (America/Cayenne) @@ -199,7 +199,7 @@ NDT -9000 D # Newfoundland Daylight Time # (America/St_Johns) NST -12600 # Newfoundland Standard Time # (America/St_Johns) -PET -18000 # Peru Time (not in zic) +PET -18000 # Peru Time (not in IANA database) PDT -25200 D # Pacific Daylight Time # (America/Dawson) # (America/Los_Angeles) @@ -269,14 +269,14 @@ BDT 21600 # Bangladesh Time # (Asia/Dhaka) BNT 28800 # Brunei Darussalam Time # (Asia/Brunei) -BORT 28800 # Borneo Time (Indonesia) (not in zic) +BORT 28800 # Borneo Time (Indonesia) (not in IANA database) BTT 21600 # Bhutan Time # (Asia/Thimphu) -CCT 28800 # China Coastal Time (not in zic) +CCT 28800 # China Coastal Time (not in IANA database) GEST Asia/Tbilisi # Georgia Summer Time (obsolete) GET Asia/Tbilisi # Georgia Time # (Asia/Tbilisi) -HKT 28800 # Hong Kong Time (not in zic) +HKT 28800 # Hong Kong Time (not in IANA database) ICT 25200 # Indochina Time # (Asia/Bangkok) # (Asia/Phnom_Penh) @@ -287,17 +287,17 @@ IDT 10800 D # Israel Daylight Time IRKST Asia/Irkutsk # Irkutsk Summer Time (obsolete) IRKT Asia/Irkutsk # Irkutsk Time # (Asia/Irkutsk) -IRT 12600 # Iran Time (not in zic) +IRT 12600 # Iran Time (not in IANA database) # CONFLICT! IST is not unique # Other timezones: # - IST: Irish Summer Time (Europe) # - IST: Indian Standard Time (Asia) IST 7200 # Israel Standard Time # (Asia/Jerusalem) -JAYT 32400 # Jayapura Time (Indonesia) (not in zic) +JAYT 32400 # Jayapura Time (Indonesia) (not in IANA database) JST 32400 # Japan Standard Time # (Asia/Tokyo) -KDT 36000 D # Korean Daylight Time (not in zic) +KDT 36000 D # Korean Daylight Time (not in IANA database) KGST 21600 D # Kyrgyzstan Summer Time (obsolete) KGT Asia/Bishkek # Kyrgyzstan Time # (Asia/Bishkek) @@ -408,7 +408,7 @@ FKT Atlantic/Stanley # Falkland Islands Time (obsolete) #################### AUSTRALIA #################### -ACSST 37800 D # Australian Central Summer Standard Time (not in zic) +ACSST 37800 D # Australian Central Summer Standard Time (not in IANA database) ACDT 37800 D # Australian Central Daylight Time # (Australia/Adelaide) # (Australia/Broken_Hill) @@ -419,7 +419,7 @@ ACST 34200 # Australian Central Standard Time # (Australia/Darwin) ACWST 31500 # Australian Central Western Standard Time # (Australia/Eucla) -AESST 39600 D # Australian Eastern Summer Standard Time (not in zic) +AESST 39600 D # Australian Eastern Summer Standard Time (not in IANA database) AEDT 39600 D # Australian Eastern Daylight Time # (Australia/Brisbane) # (Australia/Currie) @@ -434,21 +434,21 @@ AEST 36000 # Australian Eastern Standard Time # (Australia/Lindeman) # (Australia/Melbourne) # (Australia/Sydney) -AWSST 32400 D # Australia Western Summer Standard Time (not in zic) +AWSST 32400 D # Australia Western Summer Standard Time (not in IANA database) AWST 28800 # Australian Western Standard Time # (Australia/Perth) -CADT 37800 D # Central Australia Daylight-Saving Time (not in zic) -CAST 34200 # Central Australia Standard Time (not in zic) +CADT 37800 D # Central Australia Daylight-Saving Time (not in IANA database) +CAST 34200 # Central Australia Standard Time (not in IANA database) LHDT Australia/Lord_Howe # Lord Howe Daylight Time # (Australia/Lord_Howe) LHST 37800 # Lord Howe Standard Time # (Australia/Lord_Howe) -LIGT 36000 # Melbourne, Australia (not in zic) -NZT 43200 # New Zealand Time (not in zic) -SADT 37800 D # South Australian Daylight-Saving Time (not in zic) -WADT 28800 D # West Australian Daylight-Saving Time (not in zic) -WAST 25200 # West Australian Standard Time (not in zic) -WDT 32400 D # West Australian Daylight-Saving Time (not in zic) +LIGT 36000 # Melbourne, Australia (not in IANA database) +NZT 43200 # New Zealand Time (not in IANA database) +SADT 37800 D # South Australian Daylight-Saving Time (not in IANA database) +WADT 28800 D # West Australian Daylight-Saving Time (not in IANA database) +WAST 25200 # West Australian Standard Time (not in IANA database) +WDT 32400 D # West Australian Daylight-Saving Time (not in IANA database) #################### ETC #################### @@ -472,7 +472,7 @@ GMT 0 # Greenwich Mean Time # (Europe/London) UCT 0 # Universal Coordinated Time # (Etc/UCT) -UT 0 # Universal Time (not in zic) +UT 0 # Universal Time (not in IANA database) UTC 0 # Coordinated Universal Time Z 0 # Zulu ZULU 0 # Zulu @@ -631,10 +631,10 @@ EETDST 10800 D # East-Egypt Summertime FET 10800 # Further-eastern European Time (obsolete) # (Europe/Kaliningrad) # (Europe/Minsk) -MEST 7200 D # Middle Europe Summer Time (not in zic) -MET 3600 # Middle Europe Time (not in zic) -METDST 7200 D # Middle Europe Summer Time (not in zic) -MEZ 3600 # Mitteleuropaeische Zeit (German) (not in zic) +MEST 7200 D # Middle Europe Summer Time (not in IANA database) +MET 3600 # Middle Europe Time (not in IANA database) +METDST 7200 D # Middle Europe Summer Time (not in IANA database) +MEZ 3600 # Mitteleuropaeische Zeit (German) (not in IANA database) MSD 14400 D # Moscow Daylight Time (obsolete) MSK Europe/Moscow # Moscow Time # (Europe/Moscow) @@ -708,11 +708,11 @@ MART -34200 # Marquesas Time MHT 43200 # Kwajalein Time # (Pacific/Kwajalein) # (Pacific/Majuro) -MPT 36000 # North Mariana Islands Time (not in zic) +MPT 36000 # North Mariana Islands Time (not in IANA database) # CONFLICT! NFT is not unique # Other timezones: # - NFT: Norfolk Time (Pacific) -NFT -12600 # Newfoundland Time (not in zic) +NFT -12600 # Newfoundland Time (not in IANA database) NUT Pacific/Niue # Niue Time # (Pacific/Niue) NZDT 46800 D # New Zealand Daylight Time @@ -729,13 +729,13 @@ PONT 39600 # Ponape Time (Micronesia) # (Pacific/Ponape) PWT 32400 # Palau Time # (Pacific/Palau) -TAHT -36000 # Tahiti Time (zic says "TAHT", other sources "THAT") +TAHT -36000 # Tahiti Time (IANA database says "TAHT", other sources "THAT") # (Pacific/Tahiti) TKT Pacific/Fakaofo # Tokelau Time # (Pacific/Fakaofo) TOT 46800 # Tonga Time # (Pacific/Tongatapu) -TRUT 36000 # Truk Time (zic used to say "TRUT", other sources say "TRUK") +TRUT 36000 # Truk Time (IANA database used to say "TRUT", other sources say "TRUK") # (Pacific/Truk) TVT 43200 # Tuvalu Time # (Pacific/Funafuti) @@ -745,4 +745,4 @@ WAKT 43200 # Wake Time # (Pacific/Wake) WFT 43200 # Wallis and Futuna Time # (Pacific/Wallis) -YAPT 36000 # Yap Time (Micronesia) (not in zic) +YAPT 36000 # Yap Time (Micronesia) (not in IANA database) diff --git a/src/timezone/tznames/Etc.txt b/src/timezone/tznames/Etc.txt index a4ea1dee68..aa48404819 100644 --- a/src/timezone/tznames/Etc.txt +++ b/src/timezone/tznames/Etc.txt @@ -27,7 +27,7 @@ GMT 0 # Greenwich Mean Time # (Europe/London) UCT 0 # Universal Coordinated Time # (Etc/UCT) -UT 0 # Universal Time (not in zic) +UT 0 # Universal Time (not in IANA database) UTC 0 # Coordinated Universal Time # (Etc/UTC) Z 0 # Zulu diff --git a/src/timezone/tznames/Europe.txt b/src/timezone/tznames/Europe.txt index a4223e5e22..5d6a594ab5 100644 --- a/src/timezone/tznames/Europe.txt +++ b/src/timezone/tznames/Europe.txt @@ -182,10 +182,10 @@ GMT 0 # Greenwich Mean Time # - IST: Israel Standard Time (Asia) IST 3600 D # Irish Summer Time # (Europe/Dublin) -MEST 7200 D # Middle Europe Summer Time (not in zic) -MET 3600 # Middle Europe Time (not in zic) -METDST 7200 D # Middle Europe Summer Time (not in zic) -MEZ 3600 # Mitteleurop�ische Zeit (German) (not in zic) +MEST 7200 D # Middle Europe Summer Time (not in IANA database) +MET 3600 # Middle Europe Time (not in IANA database) +METDST 7200 D # Middle Europe Summer Time (not in IANA database) +MEZ 3600 # Mitteleurop�ische Zeit (German) (not in IANA database) MSD 14400 D # Moscow Daylight Time (obsolete) MSK Europe/Moscow # Moscow Time # (Europe/Moscow) diff --git a/src/timezone/tznames/Pacific.txt b/src/timezone/tznames/Pacific.txt index 67211fcad0..85c9991298 100644 --- a/src/timezone/tznames/Pacific.txt +++ b/src/timezone/tznames/Pacific.txt @@ -16,7 +16,7 @@ CHADT 49500 D # Chatham Daylight Time (New Zealand) # (Pacific/Chatham) CHAST 45900 # Chatham Standard Time (New Zealand) # (Pacific/Chatham) -ChST 36000 # Chamorro Standard Time (lower case "h" is as in zic) +ChST 36000 # Chamorro Standard Time (lower case "h" is as in IANA database) # (Pacific/Guam) # (Pacific/Saipan) CHUT 36000 # Chuuk Time @@ -52,7 +52,7 @@ MART -34200 # Marquesas Time MHT 43200 # Kwajalein Time # (Pacific/Kwajalein) # (Pacific/Majuro) -MPT 36000 # North Mariana Islands Time (not in zic) +MPT 36000 # North Mariana Islands Time (not in IANA database) NCT 39600 # New Caledonia Time # (Pacific/Noumea) # CONFLICT! NFT is not unique @@ -90,13 +90,13 @@ SBT 39600 # Solomon Islands Time SST -39600 # South Sumatran Time # (Pacific/Midway) # (Pacific/Pago_Pago) -TAHT -36000 # Tahiti Time (zic says "TAHT", other sources "THAT") +TAHT -36000 # Tahiti Time (IANA database says "TAHT", other sources "THAT") # (Pacific/Tahiti) TKT Pacific/Fakaofo # Tokelau Time # (Pacific/Fakaofo) TOT 46800 # Tonga Time # (Pacific/Tongatapu) -TRUT 36000 # Truk Time (zic used to say "TRUT", other sources say "TRUK") +TRUT 36000 # Truk Time (IANA database used to say "TRUT", other sources say "TRUK") # (Pacific/Truk) TVT 43200 # Tuvalu Time # (Pacific/Funafuti) @@ -113,5 +113,5 @@ WSST 46800 # West Samoa Standard Time # CONFLICT! WST is not unique # Other timezones: # - WST: Western Standard Time (Australia) -WST 46800 # West Samoa Time (caution: this used to mean -39600) (not in zic) -YAPT 36000 # Yap Time (Micronesia) (not in zic) +WST 46800 # West Samoa Time (caution: this used to mean -39600) (not in IANA database) +YAPT 36000 # Yap Time (Micronesia) (not in IANA database) diff --git a/src/tools/RELEASE_CHANGES b/src/tools/RELEASE_CHANGES index 4f481de12c..45cd178ae7 100644 --- a/src/tools/RELEASE_CHANGES +++ b/src/tools/RELEASE_CHANGES @@ -11,7 +11,7 @@ For All Releases (major, minor, beta, RC) o run spellchecker on result o add SGML markup -* Update timezone data to match latest zic database and new +* Update timezone data to match latest IANA timezone database and new Windows releases, if any (see src/timezone/README) * Translation updates From c7f68bea22bf680a4eab4b8b1592b3c90bc634ac Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 4 Sep 2016 20:02:16 -0400 Subject: [PATCH 111/871] Add regression test coverage for non-default timezone abbreviation sets. After further reflection about the mess cleaned up in commit 39b691f25, I decided the main bit of test coverage that was still missing was to check that the non-default abbreviation-set files we supply are usable. Add that. Back-patch to supported branches, just because it seems like a good idea to keep this all in sync. --- src/test/regress/expected/timestamptz.out | 15 +++++++++++++++ src/test/regress/sql/timestamptz.sql | 5 +++++ src/timezone/README | 5 +++++ 3 files changed, 25 insertions(+) diff --git a/src/test/regress/expected/timestamptz.out b/src/test/regress/expected/timestamptz.out index 2bfc13ad72..9fa93a43ea 100644 --- a/src/test/regress/expected/timestamptz.out +++ b/src/test/regress/expected/timestamptz.out @@ -2623,3 +2623,18 @@ select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; t (1 row) +-- Let's check the non-default timezone abbreviation sets, too +set timezone_abbreviations = 'Australia'; +select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; + ok +---- + t +(1 row) + +set timezone_abbreviations = 'India'; +select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; + ok +---- + t +(1 row) + diff --git a/src/test/regress/sql/timestamptz.sql b/src/test/regress/sql/timestamptz.sql index ce9d1c2fa1..5573e0ed77 100644 --- a/src/test/regress/sql/timestamptz.sql +++ b/src/test/regress/sql/timestamptz.sql @@ -479,3 +479,8 @@ SELECT '2007-12-09 07:30:00 UTC'::timestamptz AT TIME ZONE 'VET'; -- select count(distinct utc_offset) >= 24 as ok from pg_timezone_names; select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; +-- Let's check the non-default timezone abbreviation sets, too +set timezone_abbreviations = 'Australia'; +select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; +set timezone_abbreviations = 'India'; +select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; diff --git a/src/timezone/README b/src/timezone/README index 1df4ce1f69..f2f80c7176 100644 --- a/src/timezone/README +++ b/src/timezone/README @@ -34,6 +34,11 @@ in the same commit. Usually, if a known abbreviation has changed meaning, the appropriate fix is to make it refer to a long-form zone name instead of a fixed GMT offset. +The core regression test suite does some simple validation of the zone +data and abbreviations data (notably by checking that the pg_timezone_names +and pg_timezone_abbrevs views don't throw errors). It's worth running it +as a cross-check on proposed updates. + When there has been a new release of Windows (probably including Service Packs), the list of matching timezones need to be updated. Run the script in src/tools/win32tzlist.pl on a Windows machine running this new From b6182081be4a795d70b966be2542ad32d1ffbc20 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 4 Sep 2016 20:49:44 -0400 Subject: [PATCH 112/871] Remove duplicate code from ReorderBufferCleanupTXN(). Andres is apparently the only hacker who thinks this code is better as-is. I (tgl) follow some of his logic, but the fact that it's setting off warnings from static code analyzers seems like a sufficient reason to put the complexity into a comment rather than the code. Aleksander Alekseev Discussion: <20160404190345.54d84ee8@fujitsu> --- .../replication/logical/reorderbuffer.c | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 43b584cf7e..9b430b9acb 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -1158,17 +1158,15 @@ ReorderBufferCleanupTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) txn->base_snapshot_lsn = InvalidXLogRecPtr; } - /* delete from list of known subxacts */ - if (txn->is_known_as_subxact) - { - /* NB: nsubxacts count of parent will be too high now */ - dlist_delete(&txn->node); - } - /* delete from LSN ordered list of toplevel TXNs */ - else - { - dlist_delete(&txn->node); - } + /* + * Remove TXN from its containing list. + * + * Note: if txn->is_known_as_subxact, we are deleting the TXN from its + * parent's list of known subxacts; this leaves the parent's nsubxacts + * count too high, but we don't care. Otherwise, we are deleting the TXN + * from the LSN-ordered list of toplevel TXNs. + */ + dlist_delete(&txn->node); /* now remove reference from buffer */ hash_search(rb->by_txn, From d851bef2d60ff9345249ff67c053e37fe4b364cc Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Mon, 5 Sep 2016 09:44:38 +0100 Subject: [PATCH 113/871] Dirty replication slots when using sql interface When pg_logical_slot_get_changes(...) sets confirmed_flush_lsn to the point at which replay stopped, it doesn't dirty the replication slot. So if the replay didn't cause restart_lsn or catalog_xmin to change as well, this change will not get written out to disk. Even on a clean shutdown. If Pg crashes or restarts, a subsequent pg_logical_slot_get_changes(...) call will see the same changes already replayed since it uses the slot's confirmed_flush_lsn as the start point for fetching changes. The caller can't specify a start LSN when using the SQL interface. Mark the slot as dirty after reading changes using the SQL interface so that users won't see repeated changes after a clean shutdown. Repeated changes still occur when using the walsender interface or after an unclean shutdown. Craig Ringer --- .../replication/logical/logicalfuncs.c | 15 +++++++ src/test/recovery/Makefile | 2 + src/test/recovery/t/006_logical_decoding.pl | 40 +++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 src/test/recovery/t/006_logical_decoding.pl diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 4e4c8cdaeb..9c7be2dc7b 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -321,7 +321,22 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin * business..) */ if (ctx->reader->EndRecPtr != InvalidXLogRecPtr && confirm) + { LogicalConfirmReceivedLocation(ctx->reader->EndRecPtr); + /* + * If only the confirmed_flush_lsn has changed the slot won't get + * marked as dirty by the above. Callers on the walsender interface + * are expected to keep track of their own progress and don't need + * it written out. But SQL-interface users cannot specify their own + * start positions and it's harder for them to keep track of their + * progress, so we should make more of an effort to save it for them. + * + * Dirty the slot so it's written out at the next checkpoint. We'll + * still lose its position on crash, as documented, but it's better + * than always losing the position even on clean restart. + */ + ReplicationSlotMarkDirty(); + } /* free context, call shutdown callback */ FreeDecodingContext(ctx); diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile index 929071909a..a847952d72 100644 --- a/src/test/recovery/Makefile +++ b/src/test/recovery/Makefile @@ -18,3 +18,5 @@ check: clean distclean maintainer-clean: rm -rf tmp_check + +EXTRA_INSTALL = contrib/test_decoding diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl new file mode 100644 index 0000000000..b80a9a9415 --- /dev/null +++ b/src/test/recovery/t/006_logical_decoding.pl @@ -0,0 +1,40 @@ +# Testing of logical decoding using SQL interface and/or pg_recvlogical +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 2; + +# Initialize master node +my $node_master = get_new_node('master'); +$node_master->init(allows_streaming => 1); +$node_master->append_conf( + 'postgresql.conf', qq( +max_replication_slots = 4 +wal_level = logical +)); +$node_master->start; +my $backup_name = 'master_backup'; + +$node_master->safe_psql('postgres', qq[CREATE TABLE decoding_test(x integer, y text);]); + +$node_master->safe_psql('postgres', qq[SELECT pg_create_logical_replication_slot('test_slot', 'test_decoding');]); + +$node_master->safe_psql('postgres', qq[INSERT INTO decoding_test(x,y) SELECT s, s::text FROM generate_series(1,10) s;]); + +# Basic decoding works +my($result) = $node_master->safe_psql('postgres', qq[SELECT pg_logical_slot_get_changes('test_slot', NULL, NULL);]); +is(scalar(split /^/m, $result), 12, 'Decoding produced 12 rows inc BEGIN/COMMIT'); + +# If we immediately crash the server we might lose the progress we just made +# and replay the same changes again. But a clean shutdown should never repeat +# the same changes when we use the SQL decoding interface. +$node_master->restart('fast'); + +# There are no new writes, so the result should be empty. +$result = $node_master->safe_psql('postgres', qq[SELECT pg_logical_slot_get_changes('test_slot', NULL, NULL);]); +chomp($result); +is($result, '', 'Decoding after fast restart repeats no rows'); + +# done with the node +$node_master->stop; From ec03f4121cec6cf885bf40d9dfb53b8368251e99 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Mon, 5 Sep 2016 09:47:49 +0100 Subject: [PATCH 114/871] Document LSN acronym in WAL Internals We previously didn't mention what an LSN actually was. Simon Riggs and Michael Paquier --- doc/src/sgml/acronyms.sgml | 10 ++++++++++ doc/src/sgml/wal.sgml | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/doc/src/sgml/acronyms.sgml b/doc/src/sgml/acronyms.sgml index 38f111ef9d..bf2273fa8a 100644 --- a/doc/src/sgml/acronyms.sgml +++ b/doc/src/sgml/acronyms.sgml @@ -380,6 +380,16 @@ + + LSN + + + Log Sequence Number, see pg_lsn + and WAL Internals. + + + + MSVC diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index 503ea8a2a7..9ae6547721 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -724,6 +724,10 @@ WAL Internals + + LSN + + WAL is automatically enabled; no action is required from the administrator except ensuring that the @@ -732,6 +736,18 @@ linkend="wal-configuration">). + + WAL records are appended to the WAL + logs as each new record is written. The insert position is described by + a Log Sequence Number (LSN) that is a byte offset into + the logs, increasing monotonically with each new record. + LSN values are returned as the datatype + pg_lsn. Values can be + compared to calculate the volume of WAL data that + separates them, so they are used to measure the progress of replication + and recovery. + + WAL logs are stored in the directory pg_xlog under the data directory, as a set of From 016abf1fb8333de82a259af0cc7572a4b868023b Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Mon, 5 Sep 2016 10:38:08 +0100 Subject: [PATCH 115/871] Add debug check function LWLockHeldByMeInMode() Tests whether my process holds a lock in given mode. Add initial usage in MarkBufferDirty(). Thomas Munro --- src/backend/storage/buffer/bufmgr.c | 4 ++-- src/backend/storage/lmgr/lwlock.c | 23 ++++++++++++++++++++--- src/include/storage/lwlock.h | 1 + 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 76ade3727c..90804a3c53 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -1460,8 +1460,8 @@ MarkBufferDirty(Buffer buffer) bufHdr = GetBufferDescriptor(buffer - 1); Assert(BufferIsPinned(buffer)); - /* unfortunately we can't check if the lock is held exclusively */ - Assert(LWLockHeldByMe(BufferDescriptorGetContentLock(bufHdr))); + Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), + LW_EXCLUSIVE)); old_buf_state = pg_atomic_read_u32(&bufHdr->state); for (;;) diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 53b45d72fe..9d08de75ae 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -1880,10 +1880,9 @@ LWLockReleaseAll(void) /* - * LWLockHeldByMe - test whether my process currently holds a lock + * LWLockHeldByMe - test whether my process holds a lock in any mode * - * This is meant as debug support only. We currently do not distinguish - * whether the lock is held shared or exclusive. + * This is meant as debug support only. */ bool LWLockHeldByMe(LWLock *l) @@ -1897,3 +1896,21 @@ LWLockHeldByMe(LWLock *l) } return false; } + +/* + * LWLockHeldByMeInMode - test whether my process holds a lock in given mode + * + * This is meant as debug support only. + */ +bool +LWLockHeldByMeInMode(LWLock *l, LWLockMode mode) +{ + int i; + + for (i = 0; i < num_held_lwlocks; i++) + { + if (held_lwlocks[i].lock == l && held_lwlocks[i].mode == mode) + return true; + } + return false; +} diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 959f5f1e4d..18931eb046 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -175,6 +175,7 @@ extern void LWLockRelease(LWLock *lock); extern void LWLockReleaseClearVar(LWLock *lock, uint64 *valptr, uint64 val); extern void LWLockReleaseAll(void); extern bool LWLockHeldByMe(LWLock *lock); +extern bool LWLockHeldByMeInMode(LWLock *lock, LWLockMode mode); extern bool LWLockWaitForVar(LWLock *lock, uint64 *valptr, uint64 oldval, uint64 *newval); extern void LWLockUpdateVar(LWLock *lock, uint64 *valptr, uint64 value); From 15bc038f9bcd1a9af3f625caffafc7c20322202d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 5 Sep 2016 12:59:55 -0400 Subject: [PATCH 116/871] Relax transactional restrictions on ALTER TYPE ... ADD VALUE. To prevent possibly breaking indexes on enum columns, we must keep uncommitted enum values from getting stored in tables, unless we can be sure that any such column is new in the current transaction. Formerly, we enforced this by disallowing ALTER TYPE ... ADD VALUE from being executed at all in a transaction block, unless the target enum type had been created in the current transaction. This patch removes that restriction, and instead insists that an uncommitted enum value can't be referenced unless it belongs to an enum type created in the same transaction as the value. Per discussion, this should be a bit less onerous. It does require each function that could possibly return a new enum value to SQL operations to check this restriction, but there aren't so many of those that this seems unmaintainable. Andrew Dunstan and Tom Lane Discussion: <4075.1459088427@sss.pgh.pa.us> --- doc/src/sgml/ref/alter_type.sgml | 6 +- src/backend/commands/typecmds.c | 21 +----- src/backend/tcop/utility.c | 2 +- src/backend/utils/adt/enum.c | 104 +++++++++++++++++++++++++++++ src/backend/utils/errcodes.txt | 1 + src/include/commands/typecmds.h | 2 +- src/test/regress/expected/enum.out | 65 +++++++++++++++--- src/test/regress/sql/enum.sql | 30 +++++++-- 8 files changed, 191 insertions(+), 40 deletions(-) diff --git a/doc/src/sgml/ref/alter_type.sgml b/doc/src/sgml/ref/alter_type.sgml index 9789881a5c..aec73f6285 100644 --- a/doc/src/sgml/ref/alter_type.sgml +++ b/doc/src/sgml/ref/alter_type.sgml @@ -266,8 +266,10 @@ ALTER TYPE name ADD VALUE [ IF NOT Notes - ALTER TYPE ... ADD VALUE (the form that adds a new value to an - enum type) cannot be executed inside a transaction block. + If ALTER TYPE ... ADD VALUE (the form that adds a new value to + an enum type) is executed inside a transaction block, the new value cannot + be used until after the transaction has been committed, except in the case + that the enum type itself was created earlier in the same transaction. diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c index ce04211067..8e7be78f65 100644 --- a/src/backend/commands/typecmds.c +++ b/src/backend/commands/typecmds.c @@ -1221,7 +1221,7 @@ DefineEnum(CreateEnumStmt *stmt) * Adds a new label to an existing enum. */ ObjectAddress -AlterEnum(AlterEnumStmt *stmt, bool isTopLevel) +AlterEnum(AlterEnumStmt *stmt) { Oid enum_type_oid; TypeName *typename; @@ -1236,25 +1236,6 @@ AlterEnum(AlterEnumStmt *stmt, bool isTopLevel) if (!HeapTupleIsValid(tup)) elog(ERROR, "cache lookup failed for type %u", enum_type_oid); - /* - * Ordinarily we disallow adding values within transaction blocks, because - * we can't cope with enum OID values getting into indexes and then having - * their defining pg_enum entries go away. However, it's okay if the enum - * type was created in the current transaction, since then there can be no - * such indexes that wouldn't themselves go away on rollback. (We support - * this case because pg_dump --binary-upgrade needs it.) We test this by - * seeing if the pg_type row has xmin == current XID and is not - * HEAP_UPDATED. If it is HEAP_UPDATED, we can't be sure whether the type - * was created or only modified in this xact. So we are disallowing some - * cases that could theoretically be safe; but fortunately pg_dump only - * needs the simplest case. - */ - if (HeapTupleHeaderGetXmin(tup->t_data) == GetCurrentTransactionId() && - !(tup->t_data->t_infomask & HEAP_UPDATED)) - /* safe to do inside transaction block */ ; - else - PreventTransactionChain(isTopLevel, "ALTER TYPE ... ADD"); - /* Check it's an enum and check user has permission to ALTER the enum */ checkEnumOwner(tup); diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index ac50c2a03d..ac64135d5d 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1359,7 +1359,7 @@ ProcessUtilitySlow(Node *parsetree, break; case T_AlterEnumStmt: /* ALTER TYPE (enum) */ - address = AlterEnum((AlterEnumStmt *) parsetree, isTopLevel); + address = AlterEnum((AlterEnumStmt *) parsetree); break; case T_ViewStmt: /* CREATE VIEW */ diff --git a/src/backend/utils/adt/enum.c b/src/backend/utils/adt/enum.c index 135a54428a..47d5355027 100644 --- a/src/backend/utils/adt/enum.c +++ b/src/backend/utils/adt/enum.c @@ -19,6 +19,7 @@ #include "catalog/indexing.h" #include "catalog/pg_enum.h" #include "libpq/pqformat.h" +#include "storage/procarray.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -31,6 +32,93 @@ static Oid enum_endpoint(Oid enumtypoid, ScanDirection direction); static ArrayType *enum_range_internal(Oid enumtypoid, Oid lower, Oid upper); +/* + * Disallow use of an uncommitted pg_enum tuple. + * + * We need to make sure that uncommitted enum values don't get into indexes. + * If they did, and if we then rolled back the pg_enum addition, we'd have + * broken the index because value comparisons will not work reliably without + * an underlying pg_enum entry. (Note that removal of the heap entry + * containing an enum value is not sufficient to ensure that it doesn't appear + * in upper levels of indexes.) To do this we prevent an uncommitted row from + * being used for any SQL-level purpose. This is stronger than necessary, + * since the value might not be getting inserted into a table or there might + * be no index on its column, but it's easy to enforce centrally. + * + * However, it's okay to allow use of uncommitted values belonging to enum + * types that were themselves created in the same transaction, because then + * any such index would also be new and would go away altogether on rollback. + * (This case is required by pg_upgrade.) + * + * This function needs to be called (directly or indirectly) in any of the + * functions below that could return an enum value to SQL operations. + */ +static void +check_safe_enum_use(HeapTuple enumval_tup) +{ + TransactionId xmin; + Form_pg_enum en; + HeapTuple enumtyp_tup; + + /* + * If the row is hinted as committed, it's surely safe. This provides a + * fast path for all normal use-cases. + */ + if (HeapTupleHeaderXminCommitted(enumval_tup->t_data)) + return; + + /* + * Usually, a row would get hinted as committed when it's read or loaded + * into syscache; but just in case not, let's check the xmin directly. + */ + xmin = HeapTupleHeaderGetXmin(enumval_tup->t_data); + if (!TransactionIdIsInProgress(xmin) && + TransactionIdDidCommit(xmin)) + return; + + /* It is a new enum value, so check to see if the whole enum is new */ + en = (Form_pg_enum) GETSTRUCT(enumval_tup); + enumtyp_tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(en->enumtypid)); + if (!HeapTupleIsValid(enumtyp_tup)) + elog(ERROR, "cache lookup failed for type %u", en->enumtypid); + + /* + * We insist that the type have been created in the same (sub)transaction + * as the enum value. It would be safe to allow the type's originating + * xact to be a subcommitted child of the enum value's xact, but not vice + * versa (since we might now be in a subxact of the type's originating + * xact, which could roll back along with the enum value's subxact). The + * former case seems a sufficiently weird usage pattern as to not be worth + * spending code for, so we're left with a simple equality check. + * + * We also insist that the type's pg_type row not be HEAP_UPDATED. If it + * is, we can't tell whether the row was created or only modified in the + * apparent originating xact, so it might be older than that xact. (We do + * not worry whether the enum value is HEAP_UPDATED; if it is, we might + * think it's too new and throw an unnecessary error, but we won't allow + * an unsafe case.) + */ + if (xmin == HeapTupleHeaderGetXmin(enumtyp_tup->t_data) && + !(enumtyp_tup->t_data->t_infomask & HEAP_UPDATED)) + { + /* same (sub)transaction, so safe */ + ReleaseSysCache(enumtyp_tup); + return; + } + + /* + * There might well be other tests we could do here to narrow down the + * unsafe conditions, but for now just raise an exception. + */ + ereport(ERROR, + (errcode(ERRCODE_UNSAFE_NEW_ENUM_VALUE_USAGE), + errmsg("unsafe use of new value \"%s\" of enum type %s", + NameStr(en->enumlabel), + format_type_be(en->enumtypid)), + errhint("New enum values must be committed before they can be used."))); +} + + /* Basic I/O support */ Datum @@ -59,6 +147,9 @@ enum_in(PG_FUNCTION_ARGS) format_type_be(enumtypoid), name))); + /* check it's safe to use in SQL */ + check_safe_enum_use(tup); + /* * This comes from pg_enum.oid and stores system oids in user tables. This * oid must be preserved by binary upgrades. @@ -124,6 +215,9 @@ enum_recv(PG_FUNCTION_ARGS) format_type_be(enumtypoid), name))); + /* check it's safe to use in SQL */ + check_safe_enum_use(tup); + enumoid = HeapTupleGetOid(tup); ReleaseSysCache(tup); @@ -327,9 +421,16 @@ enum_endpoint(Oid enumtypoid, ScanDirection direction) enum_tuple = systable_getnext_ordered(enum_scan, direction); if (HeapTupleIsValid(enum_tuple)) + { + /* check it's safe to use in SQL */ + check_safe_enum_use(enum_tuple); minmax = HeapTupleGetOid(enum_tuple); + } else + { + /* should only happen with an empty enum */ minmax = InvalidOid; + } systable_endscan_ordered(enum_scan); index_close(enum_idx, AccessShareLock); @@ -490,6 +591,9 @@ enum_range_internal(Oid enumtypoid, Oid lower, Oid upper) if (left_found) { + /* check it's safe to use in SQL */ + check_safe_enum_use(enum_tuple); + if (cnt >= max) { max *= 2; diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index be924d58bd..e7bdb925ac 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -398,6 +398,7 @@ Section: Class 55 - Object Not In Prerequisite State 55006 E ERRCODE_OBJECT_IN_USE object_in_use 55P02 E ERRCODE_CANT_CHANGE_RUNTIME_PARAM cant_change_runtime_param 55P03 E ERRCODE_LOCK_NOT_AVAILABLE lock_not_available +55P04 E ERRCODE_UNSAFE_NEW_ENUM_VALUE_USAGE unsafe_new_enum_value_usage Section: Class 57 - Operator Intervention diff --git a/src/include/commands/typecmds.h b/src/include/commands/typecmds.h index e4c86f1b1d..847b770f00 100644 --- a/src/include/commands/typecmds.h +++ b/src/include/commands/typecmds.h @@ -26,7 +26,7 @@ extern void RemoveTypeById(Oid typeOid); extern ObjectAddress DefineDomain(CreateDomainStmt *stmt); extern ObjectAddress DefineEnum(CreateEnumStmt *stmt); extern ObjectAddress DefineRange(CreateRangeStmt *stmt); -extern ObjectAddress AlterEnum(AlterEnumStmt *stmt, bool isTopLevel); +extern ObjectAddress AlterEnum(AlterEnumStmt *stmt); extern ObjectAddress DefineCompositeType(RangeVar *typevar, List *coldeflist); extern Oid AssignTypeArrayOid(void); diff --git a/src/test/regress/expected/enum.out b/src/test/regress/expected/enum.out index 1a61a5b0df..d4a45a306b 100644 --- a/src/test/regress/expected/enum.out +++ b/src/test/regress/expected/enum.out @@ -560,25 +560,72 @@ DROP TYPE bogus; -- check transactional behaviour of ALTER TYPE ... ADD VALUE -- CREATE TYPE bogus AS ENUM('good'); --- check that we can't add new values to existing enums in a transaction +-- check that we can add new values to existing enums in a transaction +-- but we can't use them BEGIN; -ALTER TYPE bogus ADD VALUE 'bad'; -ERROR: ALTER TYPE ... ADD cannot run inside a transaction block +ALTER TYPE bogus ADD VALUE 'new'; +SAVEPOINT x; +SELECT 'new'::bogus; -- unsafe +ERROR: unsafe use of new value "new" of enum type bogus +LINE 1: SELECT 'new'::bogus; + ^ +HINT: New enum values must be committed before they can be used. +ROLLBACK TO x; +SELECT enum_first(null::bogus); -- safe + enum_first +------------ + good +(1 row) + +SELECT enum_last(null::bogus); -- unsafe +ERROR: unsafe use of new value "new" of enum type bogus +HINT: New enum values must be committed before they can be used. +ROLLBACK TO x; +SELECT enum_range(null::bogus); -- unsafe +ERROR: unsafe use of new value "new" of enum type bogus +HINT: New enum values must be committed before they can be used. +ROLLBACK TO x; COMMIT; +SELECT 'new'::bogus; -- now safe + bogus +------- + new +(1 row) + +SELECT enumlabel, enumsortorder +FROM pg_enum +WHERE enumtypid = 'bogus'::regtype +ORDER BY 2; + enumlabel | enumsortorder +-----------+--------------- + good | 1 + new | 2 +(2 rows) + -- check that we recognize the case where the enum already existed but was --- modified in the current txn +-- modified in the current txn; this should not be considered safe BEGIN; ALTER TYPE bogus RENAME TO bogon; ALTER TYPE bogon ADD VALUE 'bad'; -ERROR: ALTER TYPE ... ADD cannot run inside a transaction block +SELECT 'bad'::bogon; +ERROR: unsafe use of new value "bad" of enum type bogon +LINE 1: SELECT 'bad'::bogon; + ^ +HINT: New enum values must be committed before they can be used. ROLLBACK; DROP TYPE bogus; --- check that we *can* add new values to existing enums in a transaction, --- if the type is new as well +-- check that we can add new values to existing enums in a transaction +-- and use them, if the type is new as well BEGIN; -CREATE TYPE bogus AS ENUM(); -ALTER TYPE bogus ADD VALUE 'good'; +CREATE TYPE bogus AS ENUM('good'); +ALTER TYPE bogus ADD VALUE 'bad'; ALTER TYPE bogus ADD VALUE 'ugly'; +SELECT enum_range(null::bogus); + enum_range +----------------- + {good,bad,ugly} +(1 row) + ROLLBACK; -- -- Cleanup diff --git a/src/test/regress/sql/enum.sql b/src/test/regress/sql/enum.sql index 88a835e8aa..d25e8dedb6 100644 --- a/src/test/regress/sql/enum.sql +++ b/src/test/regress/sql/enum.sql @@ -262,26 +262,42 @@ DROP TYPE bogus; -- CREATE TYPE bogus AS ENUM('good'); --- check that we can't add new values to existing enums in a transaction +-- check that we can add new values to existing enums in a transaction +-- but we can't use them BEGIN; -ALTER TYPE bogus ADD VALUE 'bad'; +ALTER TYPE bogus ADD VALUE 'new'; +SAVEPOINT x; +SELECT 'new'::bogus; -- unsafe +ROLLBACK TO x; +SELECT enum_first(null::bogus); -- safe +SELECT enum_last(null::bogus); -- unsafe +ROLLBACK TO x; +SELECT enum_range(null::bogus); -- unsafe +ROLLBACK TO x; COMMIT; +SELECT 'new'::bogus; -- now safe +SELECT enumlabel, enumsortorder +FROM pg_enum +WHERE enumtypid = 'bogus'::regtype +ORDER BY 2; -- check that we recognize the case where the enum already existed but was --- modified in the current txn +-- modified in the current txn; this should not be considered safe BEGIN; ALTER TYPE bogus RENAME TO bogon; ALTER TYPE bogon ADD VALUE 'bad'; +SELECT 'bad'::bogon; ROLLBACK; DROP TYPE bogus; --- check that we *can* add new values to existing enums in a transaction, --- if the type is new as well +-- check that we can add new values to existing enums in a transaction +-- and use them, if the type is new as well BEGIN; -CREATE TYPE bogus AS ENUM(); -ALTER TYPE bogus ADD VALUE 'good'; +CREATE TYPE bogus AS ENUM('good'); +ALTER TYPE bogus ADD VALUE 'bad'; ALTER TYPE bogus ADD VALUE 'ugly'; +SELECT enum_range(null::bogus); ROLLBACK; -- From f80049f76a32858601510eaaef19ab8160e4c9b3 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 5 Sep 2016 13:09:54 -0400 Subject: [PATCH 117/871] C comment: align dashes in GroupState node header Author: Jim Nasby --- src/include/nodes/execnodes.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index a4ea1b901a..e28477d82d 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1791,7 +1791,7 @@ typedef struct SortState /* --------------------- * GroupState information - * ------------------------- + * --------------------- */ typedef struct GroupState { @@ -1810,7 +1810,7 @@ typedef struct GroupState * input group during evaluation of an Agg node's output tuple(s). We * create a second ExprContext, tmpcontext, in which to evaluate input * expressions and run the aggregate transition functions. - * ------------------------- + * --------------------- */ /* these structs are private in nodeAgg.c: */ typedef struct AggStatePerAggData *AggStatePerAgg; From c54159d44ceaba26ceda9fea1804f0de122a8f30 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 5 Sep 2016 17:06:29 -0400 Subject: [PATCH 118/871] Make locale-dependent regex character classes work for large char codes. Previously, we failed to recognize Unicode characters above U+7FF as being members of locale-dependent character classes such as [[:alpha:]]. (Actually, the same problem occurs for large pg_wchar values in any multibyte encoding, but UTF8 is the only case people have actually complained about.) It's impractical to get Spencer's original code to handle character classes or ranges containing many thousands of characters, because it insists on considering each member character individually at regex compile time, whether or not the character will ever be of interest at run time. To fix, choose a cutoff point MAX_SIMPLE_CHR below which we process characters individually as before, and deal with entire ranges or classes as single entities above that. We can actually make things cheaper than before for chars below the cutoff, because the color map can now be a simple linear array for those chars, rather than the multilevel tree structure Spencer designed. It's more expensive than before for chars above the cutoff, because we must do a binary search in a list of high chars and char ranges used in the regex pattern, plus call iswalpha() and friends for each locale-dependent character class used in the pattern. However, multibyte encodings are normally designed to give smaller codes to popular characters, so that we can expect that the slow path will be taken relatively infrequently. In any case, the speed penalty appears minor except when we have to apply iswalpha() etc. to high character codes at runtime --- and the previous coding gave wrong answers for those cases, so whether it was faster is moot. Tom Lane, reviewed by Heikki Linnakangas Discussion: <15563.1471913698@sss.pgh.pa.us> --- src/backend/regex/README | 96 +- src/backend/regex/regc_color.c | 885 ++++++++++++------ src/backend/regex/regc_cvec.c | 4 +- src/backend/regex/regc_locale.c | 89 +- src/backend/regex/regc_pg_locale.c | 36 +- src/backend/regex/regcomp.c | 62 +- src/backend/regex/regexport.c | 86 +- src/backend/regex/regprefix.c | 5 +- src/include/regex/regcustom.h | 10 + src/include/regex/regex.h | 1 - src/include/regex/regguts.h | 141 ++- .../regress/expected/regex.linux.utf8.out | 164 ++++ src/test/regress/sql/regex.linux.utf8.sql | 46 + 13 files changed, 1106 insertions(+), 519 deletions(-) create mode 100644 src/test/regress/expected/regex.linux.utf8.out create mode 100644 src/test/regress/sql/regex.linux.utf8.sql diff --git a/src/backend/regex/README b/src/backend/regex/README index 6c9f48315e..f08aab69e3 100644 --- a/src/backend/regex/README +++ b/src/backend/regex/README @@ -27,13 +27,14 @@ and similarly additional source files rege_*.c that are #include'd in regexec. This was done to avoid exposing internal symbols globally; all functions not meant to be part of the library API are static. -(Actually the above is a lie in one respect: there is one more global -symbol, pg_set_regex_collation in regcomp. It is not meant to be part of -the API, but it has to be global because both regcomp and regexec call it. -It'd be better to get rid of that, as well as the static variables it -sets, in favor of keeping the needed locale state in the regex structs. -We have not done this yet for lack of a design for how to add -application-specific state to the structs.) +(Actually the above is a lie in one respect: there are two more global +symbols, pg_set_regex_collation and pg_reg_getcolor in regcomp. These are +not meant to be part of the API, but they have to be global because both +regcomp and regexec call them. It'd be better to get rid of +pg_set_regex_collation, as well as the static variables it sets, in favor of +keeping the needed locale state in the regex structs. We have not done this +yet for lack of a design for how to add application-specific state to the +structs.) What's where in src/backend/regex/: @@ -274,28 +275,65 @@ colors: an existing color has to be subdivided. The last two of these are handled with the "struct colordesc" array and -the "colorchain" links in NFA arc structs. The color map proper (that -is, the per-character lookup array) is handled as a multi-level tree, -with each tree level indexed by one byte of a character's value. The -code arranges to not have more than one copy of bottom-level tree pages -that are all-the-same-color. - -Unfortunately, this design does not seem terribly efficient for common -cases such as a tree in which all Unicode letters are colored the same, -because there aren't that many places where we get a whole page all the -same color, except at the end of the map. (It also strikes me that given -PG's current restrictions on the range of Unicode values, we could use a -3-level rather than 4-level tree; but there's not provision for that in -regguts.h at the moment.) - -A bigger problem is that it just doesn't seem very reasonable to have to -consider each Unicode letter separately at regex parse time for a regex -such as "\w"; more than likely, a huge percentage of those codes will -never be seen at runtime. We need to fix things so that locale-based -character classes are somehow processed "symbolically" without making a -full expansion of their contents at parse time. This would mean that we'd -have to be ready to call iswalpha() at runtime, but if that only happens -for high-code-value characters, it shouldn't be a big performance hit. +the "colorchain" links in NFA arc structs. + +Ideally, we'd do the first two operations using a simple linear array +storing the current color assignment for each character code. +Unfortunately, that's not terribly workable for large charsets such as +Unicode. Our solution is to divide the color map into two parts. A simple +linear array is used for character codes up to MAX_SIMPLE_CHR, which can be +chosen large enough to include all popular characters (so that the +significantly-slower code paths about to be described are seldom invoked). +Characters above that need be considered at compile time only if they +appear explicitly in the regex pattern. We store each such mentioned +character or character range as an entry in the "colormaprange" array in +the colormap. (Overlapping ranges are split into unique subranges, so that +each range in the finished list needs only a single color that describes +all its characters.) When mapping a character above MAX_SIMPLE_CHR to a +color at runtime, we search this list of ranges explicitly. + +That's still not quite enough, though, because of locale-dependent +character classes such as [[:alpha:]]. In Unicode locales these classes +may have thousands of entries that are above MAX_SIMPLE_CHR, and we +certainly don't want to be searching large colormaprange arrays at runtime. +Nor do we even want to spend the time to initialize cvec structures that +exhaustively describe all of those characters. Our solution is to compute +exact per-character colors at regex compile time only up to MAX_SIMPLE_CHR. +For characters above that, we apply the or lookup +functions at runtime for each locale-dependent character class used in the +regex pattern, constructing a bitmap that describes which classes the +runtime character belongs to. The per-character-range data structure +mentioned above actually holds, for each range, a separate color entry +for each possible combination of character class properties. That is, +the color map for characters above MAX_SIMPLE_CHR is really a 2-D array, +whose rows correspond to high characters or character ranges that are +explicitly mentioned in the regex pattern, and whose columns correspond +to sets of the locale-dependent character classes that are used in the +regex. + +As an example, given the pattern '\w\u1234[\U0001D100-\U0001D1FF]' +(and supposing that MAX_SIMPLE_CHR is less than 0x1234), we will need +a high color map with three rows. One row is for the single character +U+1234 (represented as a single-element range), one is for the range +U+1D100..U+1D1FF, and the other row represents all remaining high +characters. The color map has two columns, one for characters that +satisfy iswalnum() and one for those that don't. + +We build this color map in parallel with scanning the regex. Each time +we detect a new explicit high character (or range) or a locale-dependent +character class, we split existing entry(s) in the high color map so that +characters we need to be able to distinguish will have distinct entries +that can be given separate colors. Often, though, single entries in the +high color map will represent very large sets of characters. + +If there are both explicit high characters/ranges and locale-dependent +character classes, we may have entries in the high color map array that +have non-WHITE colors but don't actually represent any real characters. +(For example, in a row representing a singleton range, only one of the +columns could possibly be a live entry; it's the one matching the actual +locale properties for that single character.) We don't currently make +any effort to reclaim such colors. In principle it could be done, but +it's not clear that it's worth the trouble. Detailed semantics of an NFA diff --git a/src/backend/regex/regc_color.c b/src/backend/regex/regc_color.c index 8ffc8fb797..4b72894ad3 100644 --- a/src/backend/regex/regc_color.c +++ b/src/backend/regex/regc_color.c @@ -49,10 +49,6 @@ static void initcm(struct vars * v, struct colormap * cm) { - int i; - int j; - union tree *t; - union tree *nextt; struct colordesc *cd; cm->magic = CMMAGIC; @@ -64,24 +60,40 @@ initcm(struct vars * v, cm->free = 0; cd = cm->cd; /* cm->cd[WHITE] */ + cd->nschrs = MAX_SIMPLE_CHR - CHR_MIN + 1; + cd->nuchrs = 1; cd->sub = NOSUB; cd->arcs = NULL; cd->firstchr = CHR_MIN; - cd->nchrs = CHR_MAX - CHR_MIN + 1; cd->flags = 0; - /* upper levels of tree */ - for (t = &cm->tree[0], j = NBYTS - 1; j > 0; t = nextt, j--) + cm->locolormap = (color *) + MALLOC((MAX_SIMPLE_CHR - CHR_MIN + 1) * sizeof(color)); + if (cm->locolormap == NULL) { - nextt = t + 1; - for (i = BYTTAB - 1; i >= 0; i--) - t->tptr[i] = nextt; + CERR(REG_ESPACE); + cm->cmranges = NULL; /* prevent failure during freecm */ + cm->hicolormap = NULL; + return; } - /* bottom level is solid white */ - t = &cm->tree[NBYTS - 1]; - for (i = BYTTAB - 1; i >= 0; i--) - t->tcolor[i] = WHITE; - cd->block = t; + /* this memset relies on WHITE being zero: */ + memset(cm->locolormap, WHITE, + (MAX_SIMPLE_CHR - CHR_MIN + 1) * sizeof(color)); + + memset(cm->classbits, 0, sizeof(cm->classbits)); + cm->numcmranges = 0; + cm->cmranges = NULL; + cm->maxarrayrows = 4; /* arbitrary initial allocation */ + cm->hiarrayrows = 1; /* but we have only one row/col initially */ + cm->hiarraycols = 1; + cm->hicolormap = (color *) MALLOC(cm->maxarrayrows * sizeof(color)); + if (cm->hicolormap == NULL) + { + CERR(REG_ESPACE); + return; + } + /* initialize the "all other characters" row to WHITE */ + cm->hicolormap[0] = WHITE; } /* @@ -90,117 +102,67 @@ initcm(struct vars * v, static void freecm(struct colormap * cm) { - size_t i; - union tree *cb; - cm->magic = 0; - if (NBYTS > 1) - cmtreefree(cm, cm->tree, 0); - for (i = 1; i <= cm->max; i++) /* skip WHITE */ - if (!UNUSEDCOLOR(&cm->cd[i])) - { - cb = cm->cd[i].block; - if (cb != NULL) - FREE(cb); - } if (cm->cd != cm->cdspace) FREE(cm->cd); + if (cm->locolormap != NULL) + FREE(cm->locolormap); + if (cm->cmranges != NULL) + FREE(cm->cmranges); + if (cm->hicolormap != NULL) + FREE(cm->hicolormap); } /* - * cmtreefree - free a non-terminal part of a colormap tree + * pg_reg_getcolor - slow case of GETCOLOR() */ -static void -cmtreefree(struct colormap * cm, - union tree * tree, - int level) /* level number (top == 0) of this block */ +color +pg_reg_getcolor(struct colormap * cm, chr c) { - int i; - union tree *t; - union tree *fillt = &cm->tree[level + 1]; - union tree *cb; - - assert(level < NBYTS - 1); /* this level has pointers */ - for (i = BYTTAB - 1; i >= 0; i--) + int rownum, + colnum, + low, + high; + + /* Should not be used for chrs in the locolormap */ + assert(c > MAX_SIMPLE_CHR); + + /* + * Find which row it's in. The colormapranges are in order, so we can use + * binary search. + */ + rownum = 0; /* if no match, use array row zero */ + low = 0; + high = cm->numcmranges; + while (low < high) { - t = tree->tptr[i]; - assert(t != NULL); - if (t != fillt) + int middle = low + (high - low) / 2; + const colormaprange *cmr = &cm->cmranges[middle]; + + if (c < cmr->cmin) + high = middle; + else if (c > cmr->cmax) + low = middle + 1; + else { - if (level < NBYTS - 2) - { /* more pointer blocks below */ - cmtreefree(cm, t, level + 1); - FREE(t); - } - else - { /* color block below */ - cb = cm->cd[t->tcolor[0]].block; - if (t != cb) /* not a solid block */ - FREE(t); - } + rownum = cmr->rownum; /* found a match */ + break; } } -} -/* - * setcolor - set the color of a character in a colormap - */ -static color /* previous color */ -setcolor(struct colormap * cm, - chr c, - color co) -{ - uchr uc = c; - int shift; - int level; - int b; - int bottom; - union tree *t; - union tree *newt; - union tree *fillt; - union tree *lastt; - union tree *cb; - color prev; - - assert(cm->magic == CMMAGIC); - if (CISERR() || co == COLORLESS) - return COLORLESS; - - t = cm->tree; - for (level = 0, shift = BYTBITS * (NBYTS - 1); shift > 0; - level++, shift -= BYTBITS) - { - b = (uc >> shift) & BYTMASK; - lastt = t; - t = lastt->tptr[b]; - assert(t != NULL); - fillt = &cm->tree[level + 1]; - bottom = (shift <= BYTBITS) ? 1 : 0; - cb = (bottom) ? cm->cd[t->tcolor[0]].block : fillt; - if (t == fillt || t == cb) - { /* must allocate a new block */ - newt = (union tree *) MALLOC((bottom) ? - sizeof(struct colors) : sizeof(struct ptrs)); - if (newt == NULL) - { - CERR(REG_ESPACE); - return COLORLESS; - } - if (bottom) - memcpy(VS(newt->tcolor), VS(t->tcolor), - BYTTAB * sizeof(color)); - else - memcpy(VS(newt->tptr), VS(t->tptr), - BYTTAB * sizeof(union tree *)); - t = newt; - lastt->tptr[b] = t; - } + /* + * Find which column it's in --- this is all locale-dependent. + */ + if (cm->hiarraycols > 1) + { + colnum = cclass_column_index(cm, c); + return cm->hicolormap[rownum * cm->hiarraycols + colnum]; + } + else + { + /* fast path if no relevant cclasses */ + return cm->hicolormap[rownum]; } - - b = uc & BYTMASK; - prev = t->tcolor[b]; - t->tcolor[b] = co; - return prev; } /* @@ -216,7 +178,7 @@ maxcolor(struct colormap * cm) } /* - * newcolor - find a new color (must be subject of setcolor at once) + * newcolor - find a new color (must be assigned at once) * Beware: may relocate the colordescs. */ static color /* COLORLESS for error */ @@ -278,12 +240,12 @@ newcolor(struct colormap * cm) cd = &cm->cd[cm->max]; } - cd->nchrs = 0; + cd->nschrs = 0; + cd->nuchrs = 0; cd->sub = NOSUB; cd->arcs = NULL; cd->firstchr = CHR_MIN; /* in case never set otherwise */ cd->flags = 0; - cd->block = NULL; return (color) (cd - cm->cd); } @@ -305,13 +267,9 @@ freecolor(struct colormap * cm, assert(cd->arcs == NULL); assert(cd->sub == NOSUB); - assert(cd->nchrs == 0); + assert(cd->nschrs == 0); + assert(cd->nuchrs == 0); cd->flags = FREECOL; - if (cd->block != NULL) - { - FREE(cd->block); - cd->block = NULL; /* just paranoia */ - } if ((size_t) co == cm->max) { @@ -354,17 +312,25 @@ static color pseudocolor(struct colormap * cm) { color co; + struct colordesc *cd; co = newcolor(cm); if (CISERR()) return COLORLESS; - cm->cd[co].nchrs = 1; - cm->cd[co].flags = PSEUDO; + cd = &cm->cd[co]; + cd->nschrs = 0; + cd->nuchrs = 1; /* pretend it is in the upper map */ + cd->sub = NOSUB; + cd->arcs = NULL; + cd->firstchr = CHR_MIN; + cd->flags = PSEUDO; return co; } /* * subcolor - allocate a new subcolor (if necessary) to this chr + * + * This works only for chrs that map into the low color map. */ static color subcolor(struct colormap * cm, chr c) @@ -372,7 +338,9 @@ subcolor(struct colormap * cm, chr c) color co; /* current color of c */ color sco; /* new subcolor */ - co = GETCOLOR(cm, c); + assert(c <= MAX_SIMPLE_CHR); + + co = cm->locolormap[c - CHR_MIN]; sco = newsub(cm, co); if (CISERR()) return COLORLESS; @@ -380,11 +348,37 @@ subcolor(struct colormap * cm, chr c) if (co == sco) /* already in an open subcolor */ return co; /* rest is redundant */ - cm->cd[co].nchrs--; - if (cm->cd[sco].nchrs == 0) + cm->cd[co].nschrs--; + if (cm->cd[sco].nschrs == 0) cm->cd[sco].firstchr = c; - cm->cd[sco].nchrs++; - setcolor(cm, c, sco); + cm->cd[sco].nschrs++; + cm->locolormap[c - CHR_MIN] = sco; + return sco; +} + +/* + * subcolorhi - allocate a new subcolor (if necessary) to this colormap entry + * + * This is the same processing as subcolor(), but for entries in the high + * colormap, which do not necessarily correspond to exactly one chr code. + */ +static color +subcolorhi(struct colormap * cm, color *pco) +{ + color co; /* current color of entry */ + color sco; /* new subcolor */ + + co = *pco; + sco = newsub(cm, co); + if (CISERR()) + return COLORLESS; + assert(sco != COLORLESS); + + if (co == sco) /* already in an open subcolor */ + return co; /* rest is redundant */ + cm->cd[co].nuchrs--; + cm->cd[sco].nuchrs++; + *pco = sco; return sco; } @@ -400,7 +394,8 @@ newsub(struct colormap * cm, sco = cm->cd[co].sub; if (sco == NOSUB) { /* color has no open subcolor */ - if (cm->cd[co].nchrs == 1) /* optimization */ + /* optimization: singly-referenced color need not be subcolored */ + if ((cm->cd[co].nschrs + cm->cd[co].nuchrs) == 1) return co; sco = newcolor(cm); /* must create subcolor */ if (sco == COLORLESS) @@ -417,136 +412,500 @@ newsub(struct colormap * cm, } /* - * subrange - allocate new subcolors to this range of chrs, fill in arcs + * newhicolorrow - get a new row in the hicolormap, cloning it from oldrow + * + * Returns array index of new row. Note the array might move. */ -static void -subrange(struct vars * v, - chr from, - chr to, - struct state * lp, - struct state * rp) +static int +newhicolorrow(struct colormap * cm, + int oldrow) { - uchr uf; + int newrow = cm->hiarrayrows; + color *newrowptr; int i; - assert(from <= to); + /* Assign a fresh array row index, enlarging storage if needed */ + if (newrow >= cm->maxarrayrows) + { + color *newarray; + + if (cm->maxarrayrows >= INT_MAX / (cm->hiarraycols * 2)) + { + CERR(REG_ESPACE); + return 0; + } + newarray = (color *) REALLOC(cm->hicolormap, + cm->maxarrayrows * 2 * + cm->hiarraycols * sizeof(color)); + if (newarray == NULL) + { + CERR(REG_ESPACE); + return 0; + } + cm->hicolormap = newarray; + cm->maxarrayrows *= 2; + } + cm->hiarrayrows++; + + /* Copy old row data */ + newrowptr = &cm->hicolormap[newrow * cm->hiarraycols]; + memcpy(newrowptr, + &cm->hicolormap[oldrow * cm->hiarraycols], + cm->hiarraycols * sizeof(color)); + + /* Increase color reference counts to reflect new colormap entries */ + for (i = 0; i < cm->hiarraycols; i++) + cm->cd[newrowptr[i]].nuchrs++; - /* first, align "from" on a tree-block boundary */ - uf = (uchr) from; - i = (int) (((uf + BYTTAB - 1) & (uchr) ~BYTMASK) - uf); - for (; from <= to && i > 0; i--, from++) - newarc(v->nfa, PLAIN, subcolor(v->cm, from), lp, rp); - if (from > to) /* didn't reach a boundary */ + return newrow; +} + +/* + * newhicolorcols - create a new set of columns in the high colormap + * + * Essentially, extends the 2-D array to the right with a copy of itself. + */ +static void +newhicolorcols(struct colormap * cm) +{ + color *newarray; + int r, + c; + + if (cm->hiarraycols >= INT_MAX / (cm->maxarrayrows * 2)) + { + CERR(REG_ESPACE); return; + } + newarray = (color *) REALLOC(cm->hicolormap, + cm->maxarrayrows * + cm->hiarraycols * 2 * sizeof(color)); + if (newarray == NULL) + { + CERR(REG_ESPACE); + return; + } + cm->hicolormap = newarray; - /* deal with whole blocks */ - for (; to - from >= BYTTAB; from += BYTTAB) - subblock(v, from, lp, rp); + /* Duplicate existing columns to the right, and increase ref counts */ + /* Must work backwards in the array because we realloc'd in place */ + for (r = cm->hiarrayrows - 1; r >= 0; r--) + { + color *oldrowptr = &newarray[r * cm->hiarraycols]; + color *newrowptr = &newarray[r * cm->hiarraycols * 2]; + color *newrowptr2 = newrowptr + cm->hiarraycols; + + for (c = 0; c < cm->hiarraycols; c++) + { + color co = oldrowptr[c]; - /* clean up any remaining partial table */ - for (; from <= to; from++) - newarc(v->nfa, PLAIN, subcolor(v->cm, from), lp, rp); + newrowptr[c] = newrowptr2[c] = co; + cm->cd[co].nuchrs++; + } + } + + cm->hiarraycols *= 2; } /* - * subblock - allocate new subcolors for one tree block of chrs, fill in arcs + * subcolorcvec - allocate new subcolors to cvec members, fill in arcs * - * Note: subcolors that are created during execution of this function - * will not be given a useful value of firstchr; it'll be left as CHR_MIN. - * For the current usage of firstchr in pg_regprefix, this does not matter - * because such subcolors won't occur in the common prefix of a regex. + * For each chr "c" represented by the cvec, do the equivalent of + * newarc(v->nfa, PLAIN, subcolor(v->cm, c), lp, rp); + * + * Note that in typical cases, many of the subcolors are the same. + * While newarc() would discard duplicate arc requests, we can save + * some cycles by not calling it repetitively to begin with. This is + * mechanized with the "lastsubcolor" state variable. */ static void -subblock(struct vars * v, - chr start, /* first of BYTTAB chrs */ - struct state * lp, - struct state * rp) +subcolorcvec(struct vars * v, + struct cvec * cv, + struct state * lp, + struct state * rp) { - uchr uc = start; struct colormap *cm = v->cm; - int shift; - int level; + color lastsubcolor = COLORLESS; + chr ch, + from, + to; + const chr *p; int i; - int b; - union tree *t; - union tree *cb; - union tree *fillt; - union tree *lastt; - int previ; - int ndone; - color co; - color sco; - assert((uc % BYTTAB) == 0); - - /* find its color block, making new pointer blocks as needed */ - t = cm->tree; - fillt = NULL; - for (level = 0, shift = BYTBITS * (NBYTS - 1); shift > 0; - level++, shift -= BYTBITS) - { - b = (uc >> shift) & BYTMASK; - lastt = t; - t = lastt->tptr[b]; - assert(t != NULL); - fillt = &cm->tree[level + 1]; - if (t == fillt && shift > BYTBITS) - { /* need new ptr block */ - t = (union tree *) MALLOC(sizeof(struct ptrs)); - if (t == NULL) + /* ordinary characters */ + for (p = cv->chrs, i = cv->nchrs; i > 0; p++, i--) + { + ch = *p; + subcoloronechr(v, ch, lp, rp, &lastsubcolor); + NOERR(); + } + + /* and the ranges */ + for (p = cv->ranges, i = cv->nranges; i > 0; p += 2, i--) + { + from = *p; + to = *(p + 1); + if (from <= MAX_SIMPLE_CHR) + { + /* deal with simple chars one at a time */ + chr lim = (to <= MAX_SIMPLE_CHR) ? to : MAX_SIMPLE_CHR; + + while (from <= lim) { - CERR(REG_ESPACE); - return; + color sco = subcolor(cm, from); + + NOERR(); + if (sco != lastsubcolor) + { + newarc(v->nfa, PLAIN, sco, lp, rp); + NOERR(); + lastsubcolor = sco; + } + from++; } - memcpy(VS(t->tptr), VS(fillt->tptr), - BYTTAB * sizeof(union tree *)); - lastt->tptr[b] = t; } + /* deal with any part of the range that's above MAX_SIMPLE_CHR */ + if (from < to) + subcoloronerange(v, from, to, lp, rp, &lastsubcolor); + else if (from == to) + subcoloronechr(v, from, lp, rp, &lastsubcolor); + NOERR(); } - /* special cases: fill block or solid block */ - co = t->tcolor[0]; - cb = cm->cd[co].block; - if (t == fillt || t == cb) + /* and deal with cclass if any */ + if (cv->cclasscode >= 0) { - /* either way, we want a subcolor solid block */ - sco = newsub(cm, co); - t = cm->cd[sco].block; - if (t == NULL) - { /* must set it up */ - t = (union tree *) MALLOC(sizeof(struct colors)); - if (t == NULL) + int classbit; + color *pco; + int r, + c; + + /* Enlarge array if we don't have a column bit assignment for cclass */ + if (cm->classbits[cv->cclasscode] == 0) + { + cm->classbits[cv->cclasscode] = cm->hiarraycols; + newhicolorcols(cm); + NOERR(); + } + /* Apply subcolorhi() and make arc for each entry in relevant cols */ + classbit = cm->classbits[cv->cclasscode]; + pco = cm->hicolormap; + for (r = 0; r < cm->hiarrayrows; r++) + { + for (c = 0; c < cm->hiarraycols; c++) { - CERR(REG_ESPACE); - return; + if (c & classbit) + { + color sco = subcolorhi(cm, pco); + + NOERR(); + /* add the arc if needed */ + if (sco != lastsubcolor) + { + newarc(v->nfa, PLAIN, sco, lp, rp); + NOERR(); + lastsubcolor = sco; + } + } + pco++; } - for (i = 0; i < BYTTAB; i++) - t->tcolor[i] = sco; - cm->cd[sco].block = t; } - /* find loop must have run at least once */ - lastt->tptr[b] = t; - newarc(v->nfa, PLAIN, sco, lp, rp); - cm->cd[co].nchrs -= BYTTAB; - cm->cd[sco].nchrs += BYTTAB; + } +} + +/* + * subcoloronechr - do subcolorcvec's work for a singleton chr + * + * We could just let subcoloronerange do this, but it's a bit more efficient + * if we exploit the single-chr case. Also, callers find it useful for this + * to be able to handle both low and high chr codes. + */ +static void +subcoloronechr(struct vars * v, + chr ch, + struct state * lp, + struct state * rp, + color *lastsubcolor) +{ + struct colormap *cm = v->cm; + colormaprange *newranges; + int numnewranges; + colormaprange *oldrange; + int oldrangen; + int newrow; + + /* Easy case for low chr codes */ + if (ch <= MAX_SIMPLE_CHR) + { + color sco = subcolor(cm, ch); + + NOERR(); + if (sco != *lastsubcolor) + { + newarc(v->nfa, PLAIN, sco, lp, rp); + *lastsubcolor = sco; + } + return; + } + + /* + * Potentially, we could need two more colormapranges than we have now, if + * the given chr is in the middle of some existing range. + */ + newranges = (colormaprange *) + MALLOC((cm->numcmranges + 2) * sizeof(colormaprange)); + if (newranges == NULL) + { + CERR(REG_ESPACE); return; } + numnewranges = 0; - /* general case, a mixed block to be altered */ - i = 0; - while (i < BYTTAB) + /* Ranges before target are unchanged */ + for (oldrange = cm->cmranges, oldrangen = 0; + oldrangen < cm->numcmranges; + oldrange++, oldrangen++) + { + if (oldrange->cmax >= ch) + break; + newranges[numnewranges++] = *oldrange; + } + + /* Match target chr against current range */ + if (oldrangen >= cm->numcmranges || oldrange->cmin > ch) + { + /* chr does not belong to any existing range, make a new one */ + newranges[numnewranges].cmin = ch; + newranges[numnewranges].cmax = ch; + /* row state should be cloned from the "all others" row */ + newranges[numnewranges].rownum = newrow = newhicolorrow(cm, 0); + numnewranges++; + } + else if (oldrange->cmin == oldrange->cmax) + { + /* we have an existing singleton range matching the chr */ + newranges[numnewranges++] = *oldrange; + newrow = oldrange->rownum; + /* we've now fully processed this old range */ + oldrange++, oldrangen++; + } + else { - co = t->tcolor[i]; - sco = newsub(cm, co); - newarc(v->nfa, PLAIN, sco, lp, rp); - previ = i; - do + /* chr is a subset of this existing range, must split it */ + if (ch > oldrange->cmin) + { + /* emit portion of old range before chr */ + newranges[numnewranges].cmin = oldrange->cmin; + newranges[numnewranges].cmax = ch - 1; + newranges[numnewranges].rownum = oldrange->rownum; + numnewranges++; + } + /* emit chr as singleton range, initially cloning from range */ + newranges[numnewranges].cmin = ch; + newranges[numnewranges].cmax = ch; + newranges[numnewranges].rownum = newrow = + newhicolorrow(cm, oldrange->rownum); + numnewranges++; + if (ch < oldrange->cmax) { - t->tcolor[i++] = sco; - } while (i < BYTTAB && t->tcolor[i] == co); - ndone = i - previ; - cm->cd[co].nchrs -= ndone; - cm->cd[sco].nchrs += ndone; + /* emit portion of old range after chr */ + newranges[numnewranges].cmin = ch + 1; + newranges[numnewranges].cmax = oldrange->cmax; + /* must clone the row if we are making two new ranges from old */ + newranges[numnewranges].rownum = + (ch > oldrange->cmin) ? newhicolorrow(cm, oldrange->rownum) : + oldrange->rownum; + numnewranges++; + } + /* we've now fully processed this old range */ + oldrange++, oldrangen++; + } + + /* Update colors in newrow and create arcs as needed */ + subcoloronerow(v, newrow, lp, rp, lastsubcolor); + + /* Ranges after target are unchanged */ + for (; oldrangen < cm->numcmranges; oldrange++, oldrangen++) + { + newranges[numnewranges++] = *oldrange; + } + + /* Assert our original space estimate was adequate */ + assert(numnewranges <= (cm->numcmranges + 2)); + + /* And finally, store back the updated list of ranges */ + if (cm->cmranges != NULL) + FREE(cm->cmranges); + cm->cmranges = newranges; + cm->numcmranges = numnewranges; +} + +/* + * subcoloronerange - do subcolorcvec's work for a high range + */ +static void +subcoloronerange(struct vars * v, + chr from, + chr to, + struct state * lp, + struct state * rp, + color *lastsubcolor) +{ + struct colormap *cm = v->cm; + colormaprange *newranges; + int numnewranges; + colormaprange *oldrange; + int oldrangen; + int newrow; + + /* Caller should take care of non-high-range cases */ + assert(from > MAX_SIMPLE_CHR); + assert(from < to); + + /* + * Potentially, if we have N non-adjacent ranges, we could need as many as + * 2N+1 result ranges (consider case where new range spans 'em all). + */ + newranges = (colormaprange *) + MALLOC((cm->numcmranges * 2 + 1) * sizeof(colormaprange)); + if (newranges == NULL) + { + CERR(REG_ESPACE); + return; + } + numnewranges = 0; + + /* Ranges before target are unchanged */ + for (oldrange = cm->cmranges, oldrangen = 0; + oldrangen < cm->numcmranges; + oldrange++, oldrangen++) + { + if (oldrange->cmax >= from) + break; + newranges[numnewranges++] = *oldrange; + } + + /* + * Deal with ranges that (partially) overlap the target. As we process + * each such range, increase "from" to remove the dealt-with characters + * from the target range. + */ + while (oldrangen < cm->numcmranges && oldrange->cmin <= to) + { + if (from < oldrange->cmin) + { + /* Handle portion of new range that corresponds to no old range */ + newranges[numnewranges].cmin = from; + newranges[numnewranges].cmax = oldrange->cmin - 1; + /* row state should be cloned from the "all others" row */ + newranges[numnewranges].rownum = newrow = newhicolorrow(cm, 0); + numnewranges++; + /* Update colors in newrow and create arcs as needed */ + subcoloronerow(v, newrow, lp, rp, lastsubcolor); + /* We've now fully processed the part of new range before old */ + from = oldrange->cmin; + } + + if (from <= oldrange->cmin && to >= oldrange->cmax) + { + /* old range is fully contained in new, process it in-place */ + newranges[numnewranges++] = *oldrange; + newrow = oldrange->rownum; + from = oldrange->cmax + 1; + } + else + { + /* some part of old range does not overlap new range */ + if (from > oldrange->cmin) + { + /* emit portion of old range before new range */ + newranges[numnewranges].cmin = oldrange->cmin; + newranges[numnewranges].cmax = from - 1; + newranges[numnewranges].rownum = oldrange->rownum; + numnewranges++; + } + /* emit common subrange, initially cloning from old range */ + newranges[numnewranges].cmin = from; + newranges[numnewranges].cmax = + (to < oldrange->cmax) ? to : oldrange->cmax; + newranges[numnewranges].rownum = newrow = + newhicolorrow(cm, oldrange->rownum); + numnewranges++; + if (to < oldrange->cmax) + { + /* emit portion of old range after new range */ + newranges[numnewranges].cmin = to + 1; + newranges[numnewranges].cmax = oldrange->cmax; + /* must clone the row if we are making two new ranges from old */ + newranges[numnewranges].rownum = + (from > oldrange->cmin) ? newhicolorrow(cm, oldrange->rownum) : + oldrange->rownum; + numnewranges++; + } + from = oldrange->cmax + 1; + } + /* Update colors in newrow and create arcs as needed */ + subcoloronerow(v, newrow, lp, rp, lastsubcolor); + /* we've now fully processed this old range */ + oldrange++, oldrangen++; + } + + if (from <= to) + { + /* Handle portion of new range that corresponds to no old range */ + newranges[numnewranges].cmin = from; + newranges[numnewranges].cmax = to; + /* row state should be cloned from the "all others" row */ + newranges[numnewranges].rownum = newrow = newhicolorrow(cm, 0); + numnewranges++; + /* Update colors in newrow and create arcs as needed */ + subcoloronerow(v, newrow, lp, rp, lastsubcolor); + } + + /* Ranges after target are unchanged */ + for (; oldrangen < cm->numcmranges; oldrange++, oldrangen++) + { + newranges[numnewranges++] = *oldrange; + } + + /* Assert our original space estimate was adequate */ + assert(numnewranges <= (cm->numcmranges * 2 + 1)); + + /* And finally, store back the updated list of ranges */ + if (cm->cmranges != NULL) + FREE(cm->cmranges); + cm->cmranges = newranges; + cm->numcmranges = numnewranges; +} + +/* + * subcoloronerow - do subcolorcvec's work for one new row in the high colormap + */ +static void +subcoloronerow(struct vars * v, + int rownum, + struct state * lp, + struct state * rp, + color *lastsubcolor) +{ + struct colormap *cm = v->cm; + color *pco; + int i; + + /* Apply subcolorhi() and make arc for each entry in row */ + pco = &cm->hicolormap[rownum * cm->hiarraycols]; + for (i = 0; i < cm->hiarraycols; pco++, i++) + { + color sco = subcolorhi(cm, pco); + + NOERR(); + /* make the arc if needed */ + if (sco != *lastsubcolor) + { + newarc(v->nfa, PLAIN, sco, lp, rp); + NOERR(); + *lastsubcolor = sco; + } } } @@ -575,12 +934,12 @@ okcolors(struct nfa * nfa, { /* is subcolor, let parent deal with it */ } - else if (cd->nchrs == 0) + else if (cd->nschrs == 0 && cd->nuchrs == 0) { /* parent empty, its arcs change color to subcolor */ cd->sub = NOSUB; scd = &cm->cd[sco]; - assert(scd->nchrs > 0); + assert(scd->nschrs > 0 || scd->nuchrs > 0); assert(scd->sub == sco); scd->sub = NOSUB; while ((a = cd->arcs) != NULL) @@ -597,7 +956,7 @@ okcolors(struct nfa * nfa, /* parent's arcs must gain parallel subcolor arcs */ cd->sub = NOSUB; scd = &cm->cd[sco]; - assert(scd->nchrs > 0); + assert(scd->nschrs > 0 || scd->nuchrs > 0); assert(scd->sub == sco); scd->sub = NOSUB; for (a = cd->arcs; a != NULL; a = a->colorchain) @@ -711,62 +1070,54 @@ dumpcolors(struct colormap * cm, struct colordesc *end; color co; chr c; - char *has; fprintf(f, "max %ld\n", (long) cm->max); - if (NBYTS > 1) - fillcheck(cm, cm->tree, 0, f); end = CDEND(cm); for (cd = cm->cd + 1, co = 1; cd < end; cd++, co++) /* skip 0 */ + { if (!UNUSEDCOLOR(cd)) { - assert(cd->nchrs > 0); - has = (cd->block != NULL) ? "#" : ""; + assert(cd->nschrs > 0 || cd->nuchrs > 0); if (cd->flags & PSEUDO) - fprintf(f, "#%2ld%s(ps): ", (long) co, has); + fprintf(f, "#%2ld(ps): ", (long) co); else - fprintf(f, "#%2ld%s(%2d): ", (long) co, - has, cd->nchrs); + fprintf(f, "#%2ld(%2d): ", (long) co, cd->nschrs + cd->nuchrs); /* * Unfortunately, it's hard to do this next bit more efficiently. - * - * Spencer's original coding has the loop iterating from CHR_MIN - * to CHR_MAX, but that's utterly unusable for 32-bit chr. For - * debugging purposes it seems fine to print only chr codes up to - * 1000 or so. */ - for (c = CHR_MIN; c < 1000; c++) + for (c = CHR_MIN; c <= MAX_SIMPLE_CHR; c++) if (GETCOLOR(cm, c) == co) dumpchr(c, f); fprintf(f, "\n"); } -} - -/* - * fillcheck - check proper filling of a tree - */ -static void -fillcheck(struct colormap * cm, - union tree * tree, - int level, /* level number (top == 0) of this block */ - FILE *f) -{ - int i; - union tree *t; - union tree *fillt = &cm->tree[level + 1]; - - assert(level < NBYTS - 1); /* this level has pointers */ - for (i = BYTTAB - 1; i >= 0; i--) + } + /* dump the high colormap if it contains anything interesting */ + if (cm->hiarrayrows > 1 || cm->hiarraycols > 1) { - t = tree->tptr[i]; - if (t == NULL) - fprintf(f, "NULL found in filled tree!\n"); - else if (t == fillt) + int r, + c; + const color *rowptr; + + fprintf(f, "other:\t"); + for (c = 0; c < cm->hiarraycols; c++) + { + fprintf(f, "\t%ld", (long) cm->hicolormap[c]); + } + fprintf(f, "\n"); + for (r = 0; r < cm->numcmranges; r++) { + dumpchr(cm->cmranges[r].cmin, f); + fprintf(f, ".."); + dumpchr(cm->cmranges[r].cmax, f); + fprintf(f, ":"); + rowptr = &cm->hicolormap[cm->cmranges[r].rownum * cm->hiarraycols]; + for (c = 0; c < cm->hiarraycols; c++) + { + fprintf(f, "\t%ld", (long) rowptr[c]); + } + fprintf(f, "\n"); } - else if (level < NBYTS - 2) /* more pointer blocks below */ - fillcheck(cm, t, level + 1, f); } } diff --git a/src/backend/regex/regc_cvec.c b/src/backend/regex/regc_cvec.c index 3a9e8cfbbd..50b7a4574b 100644 --- a/src/backend/regex/regc_cvec.c +++ b/src/backend/regex/regc_cvec.c @@ -34,7 +34,8 @@ /* * Notes: - * Only (selected) functions in _this_ file should treat chr* as non-constant. + * Only (selected) functions in _this_ file should treat the chr arrays + * of a cvec as non-constant. */ /* @@ -67,6 +68,7 @@ clearcvec(struct cvec * cv) assert(cv != NULL); cv->nchrs = 0; cv->nranges = 0; + cv->cclasscode = -1; return cv; } diff --git a/src/backend/regex/regc_locale.c b/src/backend/regex/regc_locale.c index 399de027cd..7cb3a40a0c 100644 --- a/src/backend/regex/regc_locale.c +++ b/src/backend/regex/regc_locale.c @@ -349,6 +349,19 @@ static const struct cname } }; +/* + * The following arrays define the valid character class names. + */ +static const char *const classNames[NUM_CCLASSES + 1] = { + "alnum", "alpha", "ascii", "blank", "cntrl", "digit", "graph", + "lower", "print", "punct", "space", "upper", "xdigit", NULL +}; + +enum classes +{ + CC_ALNUM, CC_ALPHA, CC_ASCII, CC_BLANK, CC_CNTRL, CC_DIGIT, CC_GRAPH, + CC_LOWER, CC_PRINT, CC_PUNCT, CC_SPACE, CC_UPPER, CC_XDIGIT +}; /* * We do not use the hard-wired Unicode classification tables that Tcl does. @@ -543,21 +556,6 @@ cclass(struct vars * v, /* context */ int i, index; - /* - * The following arrays define the valid character class names. - */ - - static const char *const classNames[] = { - "alnum", "alpha", "ascii", "blank", "cntrl", "digit", "graph", - "lower", "print", "punct", "space", "upper", "xdigit", NULL - }; - - enum classes - { - CC_ALNUM, CC_ALPHA, CC_ASCII, CC_BLANK, CC_CNTRL, CC_DIGIT, CC_GRAPH, - CC_LOWER, CC_PRINT, CC_PUNCT, CC_SPACE, CC_UPPER, CC_XDIGIT - }; - /* * Map the name to the corresponding enumerated value. */ @@ -593,18 +591,20 @@ cclass(struct vars * v, /* context */ * pg_ctype_get_cache so that we can cache the results. Other classes * have definitions that are hard-wired here, and for those we just * construct a transient cvec on the fly. + * + * NB: keep this code in sync with cclass_column_index(), below. */ switch ((enum classes) index) { case CC_PRINT: - cv = pg_ctype_get_cache(pg_wc_isprint); + cv = pg_ctype_get_cache(pg_wc_isprint, index); break; case CC_ALNUM: - cv = pg_ctype_get_cache(pg_wc_isalnum); + cv = pg_ctype_get_cache(pg_wc_isalnum, index); break; case CC_ALPHA: - cv = pg_ctype_get_cache(pg_wc_isalpha); + cv = pg_ctype_get_cache(pg_wc_isalpha, index); break; case CC_ASCII: /* hard-wired meaning */ @@ -625,10 +625,10 @@ cclass(struct vars * v, /* context */ addrange(cv, 0x7f, 0x9f); break; case CC_DIGIT: - cv = pg_ctype_get_cache(pg_wc_isdigit); + cv = pg_ctype_get_cache(pg_wc_isdigit, index); break; case CC_PUNCT: - cv = pg_ctype_get_cache(pg_wc_ispunct); + cv = pg_ctype_get_cache(pg_wc_ispunct, index); break; case CC_XDIGIT: @@ -646,16 +646,16 @@ cclass(struct vars * v, /* context */ } break; case CC_SPACE: - cv = pg_ctype_get_cache(pg_wc_isspace); + cv = pg_ctype_get_cache(pg_wc_isspace, index); break; case CC_LOWER: - cv = pg_ctype_get_cache(pg_wc_islower); + cv = pg_ctype_get_cache(pg_wc_islower, index); break; case CC_UPPER: - cv = pg_ctype_get_cache(pg_wc_isupper); + cv = pg_ctype_get_cache(pg_wc_isupper, index); break; case CC_GRAPH: - cv = pg_ctype_get_cache(pg_wc_isgraph); + cv = pg_ctype_get_cache(pg_wc_isgraph, index); break; } @@ -665,6 +665,47 @@ cclass(struct vars * v, /* context */ return cv; } +/* + * cclass_column_index - get appropriate high colormap column index for chr + */ +static int +cclass_column_index(struct colormap * cm, chr c) +{ + int colnum = 0; + + /* Shouldn't go through all these pushups for simple chrs */ + assert(c > MAX_SIMPLE_CHR); + + /* + * Note: we should not see requests to consider cclasses that are not + * treated as locale-specific by cclass(), above. + */ + if (cm->classbits[CC_PRINT] && pg_wc_isprint(c)) + colnum |= cm->classbits[CC_PRINT]; + if (cm->classbits[CC_ALNUM] && pg_wc_isalnum(c)) + colnum |= cm->classbits[CC_ALNUM]; + if (cm->classbits[CC_ALPHA] && pg_wc_isalpha(c)) + colnum |= cm->classbits[CC_ALPHA]; + assert(cm->classbits[CC_ASCII] == 0); + assert(cm->classbits[CC_BLANK] == 0); + assert(cm->classbits[CC_CNTRL] == 0); + if (cm->classbits[CC_DIGIT] && pg_wc_isdigit(c)) + colnum |= cm->classbits[CC_DIGIT]; + if (cm->classbits[CC_PUNCT] && pg_wc_ispunct(c)) + colnum |= cm->classbits[CC_PUNCT]; + assert(cm->classbits[CC_XDIGIT] == 0); + if (cm->classbits[CC_SPACE] && pg_wc_isspace(c)) + colnum |= cm->classbits[CC_SPACE]; + if (cm->classbits[CC_LOWER] && pg_wc_islower(c)) + colnum |= cm->classbits[CC_LOWER]; + if (cm->classbits[CC_UPPER] && pg_wc_isupper(c)) + colnum |= cm->classbits[CC_UPPER]; + if (cm->classbits[CC_GRAPH] && pg_wc_isgraph(c)) + colnum |= cm->classbits[CC_GRAPH]; + + return colnum; +} + /* * allcases - supply cvec for all case counterparts of a chr (including itself) * diff --git a/src/backend/regex/regc_pg_locale.c b/src/backend/regex/regc_pg_locale.c index 551ae7dc08..ad9d4b1961 100644 --- a/src/backend/regex/regc_pg_locale.c +++ b/src/backend/regex/regc_pg_locale.c @@ -736,7 +736,7 @@ store_match(pg_ctype_cache *pcc, pg_wchar chr1, int nchrs) * Note that the result must not be freed or modified by caller. */ static struct cvec * -pg_ctype_get_cache(pg_wc_probefunc probefunc) +pg_ctype_get_cache(pg_wc_probefunc probefunc, int cclasscode) { pg_ctype_cache *pcc; pg_wchar max_chr; @@ -770,31 +770,43 @@ pg_ctype_get_cache(pg_wc_probefunc probefunc) pcc->cv.ranges = (chr *) malloc(pcc->cv.rangespace * sizeof(chr) * 2); if (pcc->cv.chrs == NULL || pcc->cv.ranges == NULL) goto out_of_memory; + pcc->cv.cclasscode = cclasscode; /* - * Decide how many character codes we ought to look through. For C locale - * there's no need to go further than 127. Otherwise, if the encoding is - * UTF8 go up to 0x7FF, which is a pretty arbitrary cutoff but we cannot - * extend it as far as we'd like (say, 0xFFFF, the end of the Basic - * Multilingual Plane) without creating significant performance issues due - * to too many characters being fed through the colormap code. This will - * need redesign to fix reasonably, but at least for the moment we have - * all common European languages covered. Otherwise (not C, not UTF8) go - * up to 255. These limits are interrelated with restrictions discussed - * at the head of this file. + * Decide how many character codes we ought to look through. In general + * we don't go past MAX_SIMPLE_CHR; chr codes above that are handled at + * runtime using the "high colormap" mechanism. However, in C locale + * there's no need to go further than 127, and if we only have a 1-byte + * API there's no need to go further than that can handle. + * + * If it's not MAX_SIMPLE_CHR that's constraining the search, mark the + * output cvec as not having any locale-dependent behavior, since there + * will be no need to do any run-time locale checks. (The #if's here + * would always be true for production values of MAX_SIMPLE_CHR, but it's + * useful to allow it to be small for testing purposes.) */ switch (pg_regex_strategy) { case PG_REGEX_LOCALE_C: +#if MAX_SIMPLE_CHR >= 127 max_chr = (pg_wchar) 127; + pcc->cv.cclasscode = -1; +#else + max_chr = (pg_wchar) MAX_SIMPLE_CHR; +#endif break; case PG_REGEX_LOCALE_WIDE: case PG_REGEX_LOCALE_WIDE_L: - max_chr = (pg_wchar) 0x7FF; + max_chr = (pg_wchar) MAX_SIMPLE_CHR; break; case PG_REGEX_LOCALE_1BYTE: case PG_REGEX_LOCALE_1BYTE_L: +#if MAX_SIMPLE_CHR >= UCHAR_MAX max_chr = (pg_wchar) UCHAR_MAX; + pcc->cv.cclasscode = -1; +#else + max_chr = (pg_wchar) MAX_SIMPLE_CHR; +#endif break; default: max_chr = 0; /* can't get here, but keep compiler quiet */ diff --git a/src/backend/regex/regcomp.c b/src/backend/regex/regcomp.c index b211cc0a18..ed95474b3f 100644 --- a/src/backend/regex/regcomp.c +++ b/src/backend/regex/regcomp.c @@ -55,7 +55,6 @@ static void cbracket(struct vars *, struct state *, struct state *); static void brackpart(struct vars *, struct state *, struct state *); static const chr *scanplain(struct vars *); static void onechr(struct vars *, chr, struct state *, struct state *); -static void dovec(struct vars *, struct cvec *, struct state *, struct state *); static void wordchrs(struct vars *); static void processlacon(struct vars *, struct state *, struct state *, int, struct state *, struct state *); @@ -96,16 +95,19 @@ static chr chrnamed(struct vars *, const chr *, const chr *, chr); /* === regc_color.c === */ static void initcm(struct vars *, struct colormap *); static void freecm(struct colormap *); -static void cmtreefree(struct colormap *, union tree *, int); -static color setcolor(struct colormap *, chr, color); static color maxcolor(struct colormap *); static color newcolor(struct colormap *); static void freecolor(struct colormap *, color); static color pseudocolor(struct colormap *); -static color subcolor(struct colormap *, chr c); +static color subcolor(struct colormap *, chr); +static color subcolorhi(struct colormap *, color *); static color newsub(struct colormap *, color); -static void subrange(struct vars *, chr, chr, struct state *, struct state *); -static void subblock(struct vars *, chr, struct state *, struct state *); +static int newhicolorrow(struct colormap *, int); +static void newhicolorcols(struct colormap *); +static void subcolorcvec(struct vars *, struct cvec *, struct state *, struct state *); +static void subcoloronechr(struct vars *, chr, struct state *, struct state *, color *); +static void subcoloronerange(struct vars *, chr, chr, struct state *, struct state *, color *); +static void subcoloronerow(struct vars *, int, struct state *, struct state *, color *); static void okcolors(struct nfa *, struct colormap *); static void colorchain(struct colormap *, struct arc *); static void uncolorchain(struct colormap *, struct arc *); @@ -114,7 +116,6 @@ static void colorcomplement(struct nfa *, struct colormap *, int, struct state * #ifdef REG_DEBUG static void dumpcolors(struct colormap *, FILE *); -static void fillcheck(struct colormap *, union tree *, int, FILE *); static void dumpchr(chr, FILE *); #endif /* === regc_nfa.c === */ @@ -215,6 +216,7 @@ static struct cvec *range(struct vars *, chr, chr, int); static int before(chr, chr); static struct cvec *eclass(struct vars *, chr, int); static struct cvec *cclass(struct vars *, const chr *, const chr *, int); +static int cclass_column_index(struct colormap *, chr); static struct cvec *allcases(struct vars *, chr); static int cmp(const chr *, const chr *, size_t); static int casecmp(const chr *, const chr *, size_t); @@ -1467,7 +1469,7 @@ brackpart(struct vars * v, NOERR(); cv = eclass(v, startc, (v->cflags & REG_ICASE)); NOERR(); - dovec(v, cv, lp, rp); + subcolorcvec(v, cv, lp, rp); return; break; case CCLASS: @@ -1477,7 +1479,7 @@ brackpart(struct vars * v, NOERR(); cv = cclass(v, startp, endp, (v->cflags & REG_ICASE)); NOERR(); - dovec(v, cv, lp, rp); + subcolorcvec(v, cv, lp, rp); return; break; default: @@ -1523,7 +1525,7 @@ brackpart(struct vars * v, NOTE(REG_UUNPORT); cv = range(v, startc, endc, (v->cflags & REG_ICASE)); NOERR(); - dovec(v, cv, lp, rp); + subcolorcvec(v, cv, lp, rp); } /* @@ -1565,46 +1567,14 @@ onechr(struct vars * v, { if (!(v->cflags & REG_ICASE)) { - newarc(v->nfa, PLAIN, subcolor(v->cm, c), lp, rp); + color lastsubcolor = COLORLESS; + + subcoloronechr(v, c, lp, rp, &lastsubcolor); return; } /* rats, need general case anyway... */ - dovec(v, allcases(v, c), lp, rp); -} - -/* - * dovec - fill in arcs for each element of a cvec - */ -static void -dovec(struct vars * v, - struct cvec * cv, - struct state * lp, - struct state * rp) -{ - chr ch, - from, - to; - const chr *p; - int i; - - /* ordinary characters */ - for (p = cv->chrs, i = cv->nchrs; i > 0; p++, i--) - { - ch = *p; - newarc(v->nfa, PLAIN, subcolor(v->cm, ch), lp, rp); - NOERR(); - } - - /* and the ranges */ - for (p = cv->ranges, i = cv->nranges; i > 0; p += 2, i--) - { - from = *p; - to = *(p + 1); - if (from <= to) - subrange(v, from, to, lp, rp); - NOERR(); - } + subcolorcvec(v, allcases(v, c), lp, rp); } /* diff --git a/src/backend/regex/regexport.c b/src/backend/regex/regexport.c index 93da82286f..a515949bb4 100644 --- a/src/backend/regex/regexport.c +++ b/src/backend/regex/regexport.c @@ -28,10 +28,6 @@ #include "regex/regexport.h" -static void scancolormap(struct colormap * cm, int co, - union tree * t, int level, chr partial, - pg_wchar **chars, int *chars_len); - /* * Get total number of NFA states. @@ -187,10 +183,7 @@ pg_reg_colorisend(const regex_t *regex, int co) * * Note: we return -1 if the color number is invalid, or if it is a special * color (WHITE or a pseudocolor), or if the number of members is uncertain. - * The latter case cannot arise right now but is specified to allow for future - * improvements (see musings about run-time handling of higher character codes - * in regex/README). Callers should not try to extract the members if -1 is - * returned. + * Callers should not try to extract the members if -1 is returned. */ int pg_reg_getnumcharacters(const regex_t *regex, int co) @@ -205,7 +198,18 @@ pg_reg_getnumcharacters(const regex_t *regex, int co) if (cm->cd[co].flags & PSEUDO) /* also pseudocolors (BOS etc) */ return -1; - return cm->cd[co].nchrs; + /* + * If the color appears anywhere in the high colormap, treat its number of + * members as uncertain. In principle we could determine all the specific + * chrs corresponding to each such entry, but it would be expensive + * (particularly if character class tests are required) and it doesn't + * seem worth it. + */ + if (cm->cd[co].nuchrs != 0) + return -1; + + /* OK, return the known number of member chrs */ + return cm->cd[co].nschrs; } /* @@ -222,6 +226,7 @@ pg_reg_getcharacters(const regex_t *regex, int co, pg_wchar *chars, int chars_len) { struct colormap *cm; + chr c; assert(regex != NULL && regex->re_magic == REMAGIC); cm = &((struct guts *) regex->re_guts)->cmap; @@ -231,62 +236,17 @@ pg_reg_getcharacters(const regex_t *regex, int co, if (cm->cd[co].flags & PSEUDO) return; - /* Recursively search the colormap tree */ - scancolormap(cm, co, cm->tree, 0, 0, &chars, &chars_len); -} - -/* - * Recursively scan the colormap tree to find chrs belonging to color "co". - * See regex/README for info about the tree structure. - * - * t: tree block to scan - * level: level (from 0) of t - * partial: partial chr code for chrs within t - * chars, chars_len: output area - */ -static void -scancolormap(struct colormap * cm, int co, - union tree * t, int level, chr partial, - pg_wchar **chars, int *chars_len) -{ - int i; - - if (level < NBYTS - 1) - { - /* non-leaf node */ - for (i = 0; i < BYTTAB; i++) - { - /* - * We do not support search for chrs of color 0 (WHITE), so - * all-white subtrees need not be searched. These can be - * recognized because they are represented by the fill blocks in - * the colormap struct. This typically allows us to avoid - * scanning large regions of higher-numbered chrs. - */ - if (t->tptr[i] == &cm->tree[level + 1]) - continue; - - /* Recursively scan next level down */ - scancolormap(cm, co, - t->tptr[i], level + 1, - (partial | (chr) i) << BYTBITS, - chars, chars_len); - } - } - else + /* + * We need only examine the low character map; there should not be any + * matching entries in the high map. + */ + for (c = CHR_MIN; c <= MAX_SIMPLE_CHR; c++) { - /* leaf node */ - for (i = 0; i < BYTTAB; i++) + if (cm->locolormap[c - CHR_MIN] == co) { - if (t->tcolor[i] == co) - { - if (*chars_len > 0) - { - **chars = partial | (chr) i; - (*chars)++; - (*chars_len)--; - } - } + *chars++ = c; + if (--chars_len == 0) + break; } } } diff --git a/src/backend/regex/regprefix.c b/src/backend/regex/regprefix.c index 04b6449a20..f338715225 100644 --- a/src/backend/regex/regprefix.c +++ b/src/backend/regex/regprefix.c @@ -194,7 +194,10 @@ findprefix(struct cnfa * cnfa, if (thiscolor == COLORLESS) break; /* The color must be a singleton */ - if (cm->cd[thiscolor].nchrs != 1) + if (cm->cd[thiscolor].nschrs != 1) + break; + /* Must not have any high-color-map entries */ + if (cm->cd[thiscolor].nuchrs != 0) break; /* diff --git a/src/include/regex/regcustom.h b/src/include/regex/regcustom.h index 459851a7f6..04a1893c80 100644 --- a/src/include/regex/regcustom.h +++ b/src/include/regex/regcustom.h @@ -77,6 +77,16 @@ typedef unsigned uchr; /* unsigned type that will hold a chr */ */ #define CHR_IS_IN_RANGE(c) ((c) <= CHR_MAX) +/* + * MAX_SIMPLE_CHR is the cutoff between "simple" and "complicated" processing + * in the color map logic. It should usually be chosen high enough to ensure + * that all common characters are <= MAX_SIMPLE_CHR. However, very large + * values will be counterproductive since they cause more regex setup time. + * Also, small values can be helpful for testing the high-color-map logic + * with plain old ASCII input. + */ +#define MAX_SIMPLE_CHR 0x7FF /* suitable value for Unicode */ + /* functions operating on chr */ #define iscalnum(x) pg_wc_isalnum(x) #define iscalpha(x) pg_wc_isalpha(x) diff --git a/src/include/regex/regex.h b/src/include/regex/regex.h index 2f89dc9326..cc73db2547 100644 --- a/src/include/regex/regex.h +++ b/src/include/regex/regex.h @@ -172,6 +172,5 @@ extern int pg_regexec(regex_t *, const pg_wchar *, size_t, size_t, rm_detail_t * extern int pg_regprefix(regex_t *, pg_wchar **, size_t *); extern void pg_regfree(regex_t *); extern size_t pg_regerror(int, const regex_t *, char *, size_t); -extern void pg_set_regex_collation(Oid collation); #endif /* _REGEX_H_ */ diff --git a/src/include/regex/regguts.h b/src/include/regex/regguts.h index b0aa641cc4..69816f1449 100644 --- a/src/include/regex/regguts.h +++ b/src/include/regex/regguts.h @@ -127,23 +127,6 @@ #define ISBSET(uv, sn) ((uv)[(sn)/UBITS] & ((unsigned)1 << ((sn)%UBITS))) - -/* - * We dissect a chr into byts for colormap table indexing. Here we define - * a byt, which will be the same as a byte on most machines... The exact - * size of a byt is not critical, but about 8 bits is good, and extraction - * of 8-bit chunks is sometimes especially fast. - */ -#ifndef BYTBITS -#define BYTBITS 8 /* bits in a byt */ -#endif -#define BYTTAB (1<flags & FREECOL) - union tree *block; /* block of solid color, if any */ }; /* * The color map itself * - * Much of the data in the colormap struct is only used at compile time. - * However, the bulk of the space usage is in the "tree" structure, so it's - * not clear that there's much point in converting the rest to a more compact - * form when compilation is finished. + * This struct holds both data used only at compile time, and the chr to + * color mapping information, used at both compile and run time. The latter + * is the bulk of the space, so it's not really worth separating out the + * compile-only portion. + * + * Ideally, the mapping data would just be an array of colors indexed by + * chr codes; but for large character sets that's impractical. Fortunately, + * common characters have smaller codes, so we can use a simple array for chr + * codes up to MAX_SIMPLE_CHR, and do something more complex for codes above + * that, without much loss of performance. The "something more complex" is a + * 2-D array of color entries, where row indexes correspond to individual chrs + * or chr ranges that have been mentioned in the regex (with row zero + * representing all other chrs), and column indexes correspond to different + * sets of locale-dependent character classes such as "isalpha". The + * classbits[k] entry is zero if we do not care about the k'th character class + * in this regex, and otherwise it is the bit to be OR'd into the column index + * if the character in question is a member of that class. We find the color + * of a high-valued chr by identifying which colormaprange it is in to get + * the row index (use row zero if it's in none of them), identifying which of + * the interesting cclasses it's in to get the column index, and then indexing + * into the 2-D hicolormap array. + * + * The colormapranges are required to be nonempty, nonoverlapping, and to + * appear in increasing chr-value order. */ + +#define NUM_CCLASSES 13 /* must match data in regc_locale.c */ + +typedef struct colormaprange +{ + chr cmin; /* range represents cmin..cmax inclusive */ + chr cmax; + int rownum; /* row index in hicolormap array (>= 1) */ +} colormaprange; + struct colormap { int magic; @@ -233,27 +213,27 @@ struct colormap color free; /* beginning of free chain (if non-0) */ struct colordesc *cd; /* pointer to array of colordescs */ #define CDEND(cm) (&(cm)->cd[(cm)->max + 1]) + + /* mapping data for chrs <= MAX_SIMPLE_CHR: */ + color *locolormap; /* simple array indexed by chr code */ + + /* mapping data for chrs > MAX_SIMPLE_CHR: */ + int classbits[NUM_CCLASSES]; /* see comment above */ + int numcmranges; /* number of colormapranges */ + colormaprange *cmranges; /* ranges of high chrs */ + color *hicolormap; /* 2-D array of color entries */ + int maxarrayrows; /* number of array rows allocated */ + int hiarrayrows; /* number of array rows in use */ + int hiarraycols; /* number of array columns (2^N) */ + /* If we need up to NINLINECDS, we store them here to save a malloc */ -#define NINLINECDS ((size_t)10) +#define NINLINECDS ((size_t) 10) struct colordesc cdspace[NINLINECDS]; - union tree tree[NBYTS]; /* tree top, plus lower-level fill blocks */ }; -/* optimization magic to do fast chr->color mapping */ -#define B0(c) ((c) & BYTMASK) -#define B1(c) (((c)>>BYTBITS) & BYTMASK) -#define B2(c) (((c)>>(2*BYTBITS)) & BYTMASK) -#define B3(c) (((c)>>(3*BYTBITS)) & BYTMASK) -#if NBYTS == 1 -#define GETCOLOR(cm, c) ((cm)->tree->tcolor[B0(c)]) -#endif -/* beware, for NBYTS>1, GETCOLOR() is unsafe -- 2nd arg used repeatedly */ -#if NBYTS == 2 -#define GETCOLOR(cm, c) ((cm)->tree->tptr[B1(c)]->tcolor[B0(c)]) -#endif -#if NBYTS == 4 -#define GETCOLOR(cm, c) ((cm)->tree->tptr[B3(c)]->tptr[B2(c)]->tptr[B1(c)]->tcolor[B0(c)]) -#endif +/* fetch color for chr; beware of multiple evaluation of c argument */ +#define GETCOLOR(cm, c) \ + ((c) <= MAX_SIMPLE_CHR ? (cm)->locolormap[(c) - CHR_MIN] : pg_reg_getcolor(cm, c)) /* @@ -264,6 +244,11 @@ struct colormap * Representation of a set of characters. chrs[] represents individual * code points, ranges[] represents ranges in the form min..max inclusive. * + * If the cvec represents a locale-specific character class, eg [[:alpha:]], + * then the chrs[] and ranges[] arrays contain only members of that class + * up to MAX_SIMPLE_CHR (inclusive). cclasscode is set to regc_locale.c's + * code for the class, rather than being -1 as it is in an ordinary cvec. + * * Note that in cvecs gotten from newcvec() and intended to be freed by * freecvec(), both arrays of chrs are after the end of the struct, not * separately malloc'd; so chrspace and rangespace are effectively immutable. @@ -276,6 +261,7 @@ struct cvec int nranges; /* number of ranges (chr pairs) */ int rangespace; /* number of ranges allocated in ranges[] */ chr *ranges; /* pointer to vector of chr pairs */ + int cclasscode; /* value of "enum classes", or -1 */ }; @@ -489,3 +475,8 @@ struct guts int nlacons; /* size of lacons[]; note that only slots * numbered 1 .. nlacons-1 are used */ }; + + +/* prototypes for functions that are exported from regcomp.c to regexec.c */ +extern void pg_set_regex_collation(Oid collation); +extern color pg_reg_getcolor(struct colormap * cm, chr c); diff --git a/src/test/regress/expected/regex.linux.utf8.out b/src/test/regress/expected/regex.linux.utf8.out new file mode 100644 index 0000000000..7c170a99f3 --- /dev/null +++ b/src/test/regress/expected/regex.linux.utf8.out @@ -0,0 +1,164 @@ +/* + * This test is for Linux/glibc systems and others that implement proper + * locale classification of Unicode characters with high code values. + * It must be run in a database with UTF8 encoding and a Unicode-aware locale. + */ +SET client_encoding TO UTF8; +-- +-- Test the "high colormap" logic with single characters and ranges that +-- exceed the MAX_SIMPLE_CHR cutoff, here assumed to be less than U+2000. +-- +-- trivial cases: +SELECT 'aⓐ' ~ U&'a\24D0' AS t; + t +--- + t +(1 row) + +SELECT 'aⓐ' ~ U&'a\24D1' AS f; + f +--- + f +(1 row) + +SELECT 'aⓕ' ~ 'a[ⓐ-ⓩ]' AS t; + t +--- + t +(1 row) + +SELECT 'aⒻ' ~ 'a[ⓐ-ⓩ]' AS f; + f +--- + f +(1 row) + +-- cases requiring splitting of ranges: +SELECT 'aⓕⓕ' ~ 'aⓕ[ⓐ-ⓩ]' AS t; + t +--- + t +(1 row) + +SELECT 'aⓕⓐ' ~ 'aⓕ[ⓐ-ⓩ]' AS t; + t +--- + t +(1 row) + +SELECT 'aⓐⓕ' ~ 'aⓕ[ⓐ-ⓩ]' AS f; + f +--- + f +(1 row) + +SELECT 'aⓕⓕ' ~ 'a[ⓐ-ⓩ]ⓕ' AS t; + t +--- + t +(1 row) + +SELECT 'aⓕⓐ' ~ 'a[ⓐ-ⓩ]ⓕ' AS f; + f +--- + f +(1 row) + +SELECT 'aⓐⓕ' ~ 'a[ⓐ-ⓩ]ⓕ' AS t; + t +--- + t +(1 row) + +SELECT 'aⒶⓜ' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS t; + t +--- + t +(1 row) + +SELECT 'aⓜⓜ' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS t; + t +--- + t +(1 row) + +SELECT 'aⓜⓩ' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS t; + t +--- + t +(1 row) + +SELECT 'aⓩⓩ' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS f; + f +--- + f +(1 row) + +SELECT 'aⓜ⓪' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS f; + f +--- + f +(1 row) + +SELECT 'a0' ~ 'a[a-ⓩ]' AS f; + f +--- + f +(1 row) + +SELECT 'aq' ~ 'a[a-ⓩ]' AS t; + t +--- + t +(1 row) + +SELECT 'aⓜ' ~ 'a[a-ⓩ]' AS t; + t +--- + t +(1 row) + +SELECT 'a⓪' ~ 'a[a-ⓩ]' AS f; + f +--- + f +(1 row) + +-- Locale-dependent character classes +SELECT 'aⒶⓜ⓪' ~ '[[:alpha:]][[:alpha:]][[:alpha:]][[:graph:]]' AS t; + t +--- + t +(1 row) + +SELECT 'aⒶⓜ⓪' ~ '[[:alpha:]][[:alpha:]][[:alpha:]][[:alpha:]]' AS f; + f +--- + f +(1 row) + +-- Locale-dependent character classes with high ranges +SELECT 'aⒶⓜ⓪' ~ '[a-z][[:alpha:]][ⓐ-ⓩ][[:graph:]]' AS t; + t +--- + t +(1 row) + +SELECT 'aⓜⒶ⓪' ~ '[a-z][[:alpha:]][ⓐ-ⓩ][[:graph:]]' AS f; + f +--- + f +(1 row) + +SELECT 'aⓜⒶ⓪' ~ '[a-z][ⓐ-ⓩ][[:alpha:]][[:graph:]]' AS t; + t +--- + t +(1 row) + +SELECT 'aⒶⓜ⓪' ~ '[a-z][ⓐ-ⓩ][[:alpha:]][[:graph:]]' AS f; + f +--- + f +(1 row) + diff --git a/src/test/regress/sql/regex.linux.utf8.sql b/src/test/regress/sql/regex.linux.utf8.sql new file mode 100644 index 0000000000..4577811645 --- /dev/null +++ b/src/test/regress/sql/regex.linux.utf8.sql @@ -0,0 +1,46 @@ +/* + * This test is for Linux/glibc systems and others that implement proper + * locale classification of Unicode characters with high code values. + * It must be run in a database with UTF8 encoding and a Unicode-aware locale. + */ + +SET client_encoding TO UTF8; + +-- +-- Test the "high colormap" logic with single characters and ranges that +-- exceed the MAX_SIMPLE_CHR cutoff, here assumed to be less than U+2000. +-- + +-- trivial cases: +SELECT 'aⓐ' ~ U&'a\24D0' AS t; +SELECT 'aⓐ' ~ U&'a\24D1' AS f; +SELECT 'aⓕ' ~ 'a[ⓐ-ⓩ]' AS t; +SELECT 'aⒻ' ~ 'a[ⓐ-ⓩ]' AS f; +-- cases requiring splitting of ranges: +SELECT 'aⓕⓕ' ~ 'aⓕ[ⓐ-ⓩ]' AS t; +SELECT 'aⓕⓐ' ~ 'aⓕ[ⓐ-ⓩ]' AS t; +SELECT 'aⓐⓕ' ~ 'aⓕ[ⓐ-ⓩ]' AS f; +SELECT 'aⓕⓕ' ~ 'a[ⓐ-ⓩ]ⓕ' AS t; +SELECT 'aⓕⓐ' ~ 'a[ⓐ-ⓩ]ⓕ' AS f; +SELECT 'aⓐⓕ' ~ 'a[ⓐ-ⓩ]ⓕ' AS t; +SELECT 'aⒶⓜ' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS t; +SELECT 'aⓜⓜ' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS t; +SELECT 'aⓜⓩ' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS t; +SELECT 'aⓩⓩ' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS f; +SELECT 'aⓜ⓪' ~ 'a[Ⓐ-ⓜ][ⓜ-ⓩ]' AS f; +SELECT 'a0' ~ 'a[a-ⓩ]' AS f; +SELECT 'aq' ~ 'a[a-ⓩ]' AS t; +SELECT 'aⓜ' ~ 'a[a-ⓩ]' AS t; +SELECT 'a⓪' ~ 'a[a-ⓩ]' AS f; + +-- Locale-dependent character classes + +SELECT 'aⒶⓜ⓪' ~ '[[:alpha:]][[:alpha:]][[:alpha:]][[:graph:]]' AS t; +SELECT 'aⒶⓜ⓪' ~ '[[:alpha:]][[:alpha:]][[:alpha:]][[:alpha:]]' AS f; + +-- Locale-dependent character classes with high ranges + +SELECT 'aⒶⓜ⓪' ~ '[a-z][[:alpha:]][ⓐ-ⓩ][[:graph:]]' AS t; +SELECT 'aⓜⒶ⓪' ~ '[a-z][[:alpha:]][ⓐ-ⓩ][[:graph:]]' AS f; +SELECT 'aⓜⒶ⓪' ~ '[a-z][ⓐ-ⓩ][[:alpha:]][[:graph:]]' AS t; +SELECT 'aⒶⓜ⓪' ~ '[a-z][ⓐ-ⓩ][[:alpha:]][[:graph:]]' AS f; From 2093f6630500c9d5e122748ac7d3719855d71174 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Mon, 5 Sep 2016 18:44:36 -0300 Subject: [PATCH 119/871] Have "make coverage" recurse into contrib as well --- GNUmakefile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GNUmakefile.in b/GNUmakefile.in index 15fba9fce0..dc76a5d11d 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -39,9 +39,9 @@ install-world: # build src/ before contrib/ install-world-contrib-recurse: install-world-src-recurse -$(call recurse,installdirs uninstall coverage init-po update-po,doc src config) +$(call recurse,installdirs uninstall init-po update-po,doc src config) -$(call recurse,distprep,doc src config contrib) +$(call recurse,distprep coverage,doc src config contrib) # clean, distclean, etc should apply to contrib too, even though # it's not built by default From 25794e841e5b86a0f90fac7f7f851e5d950e51e2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 5 Sep 2016 18:53:25 -0400 Subject: [PATCH 120/871] Cosmetic code cleanup in commands/extension.c. Some of the comments added by the CREATE EXTENSION CASCADE patch were a bit sloppy, and I didn't care for redeclaring the same local variable inside a nested block either. No functional changes. --- src/backend/commands/extension.c | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c index 518fefcf09..fa861e670b 100644 --- a/src/backend/commands/extension.c +++ b/src/backend/commands/extension.c @@ -1169,10 +1169,10 @@ find_update_path(List *evi_list, /* * CREATE EXTENSION worker * - * When CASCADE is specified CreateExtensionInternal() recurses if required - * extensions need to be installed. To sanely handle cyclic dependencies - * cascade_parent contains the dependency chain leading to the current - * invocation; thus allowing to error out if there's a cyclic dependency. + * When CASCADE is specified, CreateExtensionInternal() recurses if required + * extensions need to be installed. To sanely handle cyclic dependencies, + * the "parents" list contains a list of names of extensions already being + * installed, allowing us to error out if we recurse to one of those. */ static ObjectAddress CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) @@ -1400,8 +1400,8 @@ CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) */ /* - * Look up the prerequisite extensions, and build lists of their OIDs and - * the OIDs of their target schemas. + * Look up the prerequisite extensions, install them if necessary, and + * build lists of their OIDs and the OIDs of their target schemas. */ requiredExtensions = NIL; requiredSchemas = NIL; @@ -1416,18 +1416,19 @@ CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) { if (cascade) { + /* Must install it. */ CreateExtensionStmt *ces; - ListCell *lc; + ListCell *lc2; ObjectAddress addr; List *cascade_parents; - /* Check extension name validity before trying to cascade */ + /* Check extension name validity before trying to cascade. */ check_valid_extension_name(curreq); /* Check for cyclic dependency between extensions. */ - foreach(lc, parents) + foreach(lc2, parents) { - char *pname = (char *) lfirst(lc); + char *pname = (char *) lfirst(lc2); if (strcmp(pname, curreq) == 0) ereport(ERROR, @@ -1440,26 +1441,26 @@ CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) (errmsg("installing required extension \"%s\"", curreq))); - /* Create and execute new CREATE EXTENSION statement. */ + /* Build a CREATE EXTENSION statement to pass down. */ ces = makeNode(CreateExtensionStmt); ces->extname = curreq; + ces->if_not_exists = false; - /* Propagate the CASCADE option */ + /* Propagate the CASCADE option. */ ces->options = list_make1(d_cascade); /* Propagate the SCHEMA option if given. */ if (d_schema && d_schema->arg) ces->options = lappend(ces->options, d_schema); - /* - * Pass the current list of parents + the current extension to - * the "child" CreateExtensionInternal(). - */ + /* Add current extension to list of parents to pass down. */ cascade_parents = lappend(list_copy(parents), stmt->extname); /* Create the required extension. */ addr = CreateExtensionInternal(ces, cascade_parents); + + /* Get its newly-assigned OID. */ reqext = addr.objectId; } else @@ -1551,7 +1552,6 @@ CreateExtension(CreateExtensionStmt *stmt) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("nested CREATE EXTENSION is not supported"))); - /* Finally create the extension. */ return CreateExtensionInternal(stmt, NIL); } From 67e1e2aaff109c507da013c66009f048dda96f3e Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 6 Sep 2016 00:03:55 -0400 Subject: [PATCH 121/871] C comment: fix file name mention on line 1 Author: Amit Langote --- src/include/libpq/libpq-be.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index ecdfbc60bb..b91eca5b2c 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -1,6 +1,6 @@ /*------------------------------------------------------------------------- * - * libpq_be.h + * libpq-be.h * This file contains definitions for structures and externs used * by the postmaster during client authentication. * From dcb12ce8d8691e0e526d3f38d14c4d7fc9c664f5 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Tue, 6 Sep 2016 15:35:47 +0100 Subject: [PATCH 122/871] Fix VACUUM_TRUNCATE_LOCK_WAIT_INTERVAL lazy_truncate_heap() was waiting for VACUUM_TRUNCATE_LOCK_WAIT_INTERVAL, but in microseconds not milliseconds as originally intended. Found by code inspection. Simon Riggs --- src/backend/commands/vacuumlazy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.c index 231e92d8e4..b5fb325007 100644 --- a/src/backend/commands/vacuumlazy.c +++ b/src/backend/commands/vacuumlazy.c @@ -1747,7 +1747,7 @@ lazy_truncate_heap(Relation onerel, LVRelStats *vacrelstats) return; } - pg_usleep(VACUUM_TRUNCATE_LOCK_WAIT_INTERVAL); + pg_usleep(VACUUM_TRUNCATE_LOCK_WAIT_INTERVAL * 1000L); } /* From a2ee579b6def8e0bde876c6c2fc9d4b8ec2b6b67 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 6 Sep 2016 13:26:43 -0400 Subject: [PATCH 123/871] Repair whitespace in initdb message. What used to be four spaces somehow turned into a tab and a couple of spaces in commit a00c58314, no doubt from overhelpful emacs autoindent. Noted by Peter Eisentraut. --- src/bin/initdb/initdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 94074928cb..3350e13059 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -3604,7 +3604,7 @@ main(int argc, char *argv[]) appendPQExpBufferStr(start_db_cmd, " -l logfile start"); printf(_("\nSuccess. You can now start the database server using:\n\n" - " %s\n\n"), + " %s\n\n"), start_db_cmd->data); destroyPQExpBuffer(start_db_cmd); From cdc70597c9ba62aad08a46e55c0c783bf4c21c9c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 6 Sep 2016 14:53:31 -0400 Subject: [PATCH 124/871] Teach appendShellString() to not quote strings containing "-". Brain fade in commit a00c58314: I was thinking that a string starting with "-" could be taken as a switch depending on command line syntax. That's true, but having appendShellString() quote it will not help, so we may as well not do so. Per complaint from Peter Eisentraut. --- src/fe_utils/string_utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fe_utils/string_utils.c b/src/fe_utils/string_utils.c index edbc869e45..61bf9e6ca6 100644 --- a/src/fe_utils/string_utils.c +++ b/src/fe_utils/string_utils.c @@ -439,7 +439,7 @@ appendShellString(PQExpBuffer buf, const char *str) * contains only safe characters. */ if (*str != '\0' && - strspn(str, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./:") == strlen(str)) + strspn(str, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_./:") == strlen(str)) { appendPQExpBufferStr(buf, str); return; From f032722f86a709277d44a317a3bc8acd74861a72 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 6 Sep 2016 15:50:31 -0400 Subject: [PATCH 125/871] Guard against possible memory allocation botch in batchmemtuples(). Negative availMemLessRefund would be problematic. It's not entirely clear whether the case can be hit in the code as it stands, but this seems like good future-proofing in any case. While we're at it, insist that the value be not merely positive but not tiny, so as to avoid doing a lot of repalloc work for little gain. Peter Geoghegan Discussion: --- src/backend/utils/sort/tuplesort.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index c8fbcf8fcc..aa8e0e42fc 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -2866,6 +2866,9 @@ batchmemtuples(Tuplesortstate *state) int64 availMemLessRefund; int memtupsize = state->memtupsize; + /* Caller error if we have no tapes */ + Assert(state->activeTapes > 0); + /* For simplicity, assume no memtuples are actually currently counted */ Assert(state->memtupcount == 0); @@ -2879,6 +2882,20 @@ batchmemtuples(Tuplesortstate *state) refund = memtupsize * STANDARDCHUNKHEADERSIZE; availMemLessRefund = state->availMem - refund; + /* + * We need to be sure that we do not cause LACKMEM to become true, else + * the batch allocation size could be calculated as negative, causing + * havoc. Hence, if availMemLessRefund is negative at this point, we must + * do nothing. Moreover, if it's positive but rather small, there's + * little point in proceeding because we could only increase memtuples by + * a small amount, not worth the cost of the repalloc's. We somewhat + * arbitrarily set the threshold at ALLOCSET_DEFAULT_INITSIZE per tape. + * (Note that this does not represent any assumption about tuple sizes.) + */ + if (availMemLessRefund <= + (int64) state->activeTapes * ALLOCSET_DEFAULT_INITSIZE) + return; + /* * To establish balanced memory use after refunding palloc overhead, * temporarily have our accounting indicate that we've allocated all @@ -2888,9 +2905,11 @@ batchmemtuples(Tuplesortstate *state) state->growmemtuples = true; USEMEM(state, availMemLessRefund); (void) grow_memtuples(state); - /* Should not matter, but be tidy */ - FREEMEM(state, availMemLessRefund); state->growmemtuples = false; + /* availMem must stay accurate for spacePerTape calculation */ + FREEMEM(state, availMemLessRefund); + if (LACKMEM(state)) + elog(ERROR, "unexpected out-of-memory situation in tuplesort"); #ifdef TRACE_SORT if (trace_sort) From 975768f8eae2581b89ceafe8b16a77ff375207fe Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 6 Sep 2016 17:50:53 -0400 Subject: [PATCH 126/871] Doc: small improvements for documentation about VACUUM freezing. Mostly, explain how row xmin's used to be replaced by FrozenTransactionId and no longer are. Do a little copy-editing on the side. Per discussion with Egor Rogov. Back-patch to 9.4 where the behavioral change occurred. Discussion: <575D7955.6060209@postgrespro.ru> --- doc/src/sgml/maintenance.sgml | 48 +++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index 2713883019..f87f3e00de 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -389,7 +389,8 @@ - PostgreSQL's MVCC transaction semantics + PostgreSQL's + MVCC transaction semantics depend on being able to compare transaction ID (XID) numbers: a row version with an insertion XID greater than the current transaction's XID is in the future and should not be visible @@ -407,13 +408,10 @@ The reason that periodic vacuuming solves the problem is that VACUUM will mark rows as frozen, indicating that - they were inserted by a transaction which committed sufficiently far in - the past that the effects of the inserting transaction is certain to be - visible, from an MVCC perspective, to all current and future transactions. - PostgreSQL reserves a special XID, - FrozenTransactionId, which does not follow the normal XID - comparison rules and is always considered older - than every normal XID. Normal XIDs are + they were inserted by a transaction that committed sufficiently far in + the past that the effects of the inserting transaction are certain to be + visible to all current and future transactions. + Normal XIDs are compared using modulo-232 arithmetic. This means that for every normal XID, there are two billion XIDs that are older and two billion that are newer; another @@ -423,16 +421,40 @@ the next two billion transactions, no matter which normal XID we are talking about. If the row version still exists after more than two billion transactions, it will suddenly appear to be in the future. To - prevent this, frozen row versions are treated as if the inserting XID were + prevent this, PostgreSQL reserves a special XID, + FrozenTransactionId, which does not follow the normal XID + comparison rules and is always considered older + than every normal XID. + Frozen row versions are treated as if the inserting XID were FrozenTransactionId, so that they will appear to be in the past to all normal transactions regardless of wraparound issues, and so such row versions will be valid until deleted, no matter how long that is. + + + In PostgreSQL versions before 9.4, freezing was + implemented by actually replacing a row's insertion XID + with FrozenTransactionId, which was visible in the + row's xmin system column. Newer versions just set a flag + bit, preserving the row's original xmin for possible + forensic use. However, rows with xmin equal + to FrozenTransactionId (2) may still be found + in databases pg_upgrade'd from pre-9.4 versions. + + + Also, system catalogs may contain rows with xmin equal + to BootstrapTransactionId (1), indicating that they were + inserted during the first phase of initdb. + Like FrozenTransactionId, this special XID is treated as + older than every normal XID. + + + - controls how old an XID value has to be before its row version will be + controls how old an XID value has to be before rows bearing that XID will be frozen. Increasing this setting may avoid unnecessary work if the rows that would otherwise be frozen will soon be modified again, but decreasing this setting increases @@ -442,10 +464,10 @@ VACUUM uses the visibility map - to determine which pages of a relation must be scanned. Normally, it - will skips pages that don't have any dead row versions even if those pages + to determine which pages of a table must be scanned. Normally, it + will skip pages that don't have any dead row versions even if those pages might still have row versions with old XID values. Therefore, normal - scans won't succeed in freezing every row version in the table. + VACUUMs won't always freeze every old row version in the table. Periodically, VACUUM will perform an aggressive vacuum, skipping only those pages which contain neither dead rows nor any unfrozen XID or MXID values. From 49eb0fd0972d14014dd3533b1f1bf8c94c899883 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 6 Sep 2016 12:00:00 -0400 Subject: [PATCH 127/871] Add location field to DefElem Add a location field to the DefElem struct, used to parse many utility commands. Update various error messages to supply error position information. To propogate the error position information in a more systematic way, create a ParseState in standard_ProcessUtility() and pass that to interested functions implementing the utility commands. This seems better than passing the query string and then reassembling a parse state ad hoc, which violates the encapsulation of the ParseState type. Reviewed-by: Pavel Stehule --- contrib/file_fdw/file_fdw.c | 16 +- src/backend/access/common/reloptions.c | 2 +- src/backend/catalog/aclchk.c | 8 +- src/backend/commands/aggregatecmds.c | 7 +- src/backend/commands/collationcmds.c | 5 +- src/backend/commands/copy.c | 93 ++++--- src/backend/commands/dbcommands.c | 61 +++-- src/backend/commands/define.c | 9 - src/backend/commands/explain.c | 8 +- src/backend/commands/extension.c | 25 +- src/backend/commands/functioncmds.c | 57 ++-- src/backend/commands/sequence.c | 36 ++- src/backend/commands/tsearchcmds.c | 8 +- src/backend/commands/typecmds.c | 8 +- src/backend/commands/user.c | 41 ++- src/backend/commands/view.c | 4 +- src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/equalfuncs.c | 2 + src/backend/nodes/makefuncs.c | 6 +- src/backend/nodes/outfuncs.c | 1 + src/backend/nodes/readfuncs.c | 1 + src/backend/parser/gram.y | 248 +++++++++--------- src/backend/parser/parse_utilcmd.c | 5 +- .../replication/logical/logicalfuncs.c | 2 +- src/backend/replication/repl_gram.y | 16 +- src/backend/tcop/utility.c | 64 +++-- src/include/commands/collationcmds.h | 2 +- src/include/commands/copy.h | 7 +- src/include/commands/dbcommands.h | 4 +- src/include/commands/defrem.h | 13 +- src/include/commands/explain.h | 3 +- src/include/commands/extension.h | 4 +- src/include/commands/sequence.h | 5 +- src/include/commands/typecmds.h | 2 +- src/include/commands/user.h | 3 +- src/include/nodes/makefuncs.h | 4 +- src/include/nodes/parsenodes.h | 1 + src/include/utils/acl.h | 3 +- 38 files changed, 438 insertions(+), 347 deletions(-) diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index b42de873e0..b471991318 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -293,7 +293,7 @@ file_fdw_validator(PG_FUNCTION_ARGS) /* * Now apply the core COPY code's validation logic for more checks. */ - ProcessCopyOptions(NULL, true, other_options); + ProcessCopyOptions(NULL, NULL, true, other_options); /* * Filename option is required for file_fdw foreign tables. @@ -455,10 +455,10 @@ get_file_fdw_attribute_options(Oid relid) * force_null options set */ if (fnncolumns != NIL) - options = lappend(options, makeDefElem("force_not_null", (Node *) fnncolumns)); + options = lappend(options, makeDefElem("force_not_null", (Node *) fnncolumns, -1)); if (fncolumns != NIL) - options = lappend(options, makeDefElem("force_null", (Node *) fncolumns)); + options = lappend(options, makeDefElem("force_null", (Node *) fncolumns, -1)); return options; } @@ -511,7 +511,7 @@ fileGetForeignPaths(PlannerInfo *root, foreigntableid, &columns)) coptions = list_make1(makeDefElem("convert_selectively", - (Node *) columns)); + (Node *) columns, -1)); /* Estimate costs */ estimate_costs(root, baserel, fdw_private, @@ -632,7 +632,8 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) * Create CopyState from FDW options. We always acquire all columns, so * as to match the expected ScanTupleSlot signature. */ - cstate = BeginCopyFrom(node->ss.ss_currentRelation, + cstate = BeginCopyFrom(NULL, + node->ss.ss_currentRelation, filename, false, NIL, @@ -705,7 +706,8 @@ fileReScanForeignScan(ForeignScanState *node) EndCopyFrom(festate->cstate); - festate->cstate = BeginCopyFrom(node->ss.ss_currentRelation, + festate->cstate = BeginCopyFrom(NULL, + node->ss.ss_currentRelation, festate->filename, false, NIL, @@ -1053,7 +1055,7 @@ file_acquire_sample_rows(Relation onerel, int elevel, /* * Create CopyState from FDW options. */ - cstate = BeginCopyFrom(onerel, filename, false, NIL, options); + cstate = BeginCopyFrom(NULL, onerel, filename, false, NIL, options); /* * Use per-tuple memory context to prevent leak of memory used to read diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index ba1f3aafed..83a97b06ab 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -888,7 +888,7 @@ untransformRelOptions(Datum options) *p++ = '\0'; val = (Node *) makeString(pstrdup(p)); } - result = lappend(result, makeDefElem(pstrdup(s), val)); + result = lappend(result, makeDefElem(pstrdup(s), val, -1)); } return result; diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index a585c3ad19..c0df6710d1 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -849,7 +849,7 @@ getRelationsInNamespace(Oid namespaceId, char relkind) * ALTER DEFAULT PRIVILEGES statement */ void -ExecAlterDefaultPrivilegesStmt(AlterDefaultPrivilegesStmt *stmt) +ExecAlterDefaultPrivilegesStmt(ParseState *pstate, AlterDefaultPrivilegesStmt *stmt) { GrantStmt *action = stmt->action; InternalDefaultACL iacls; @@ -871,7 +871,8 @@ ExecAlterDefaultPrivilegesStmt(AlterDefaultPrivilegesStmt *stmt) if (dnspnames) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dnspnames = defel; } else if (strcmp(defel->defname, "roles") == 0) @@ -879,7 +880,8 @@ ExecAlterDefaultPrivilegesStmt(AlterDefaultPrivilegesStmt *stmt) if (drolespecs) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); drolespecs = defel; } else diff --git a/src/backend/commands/aggregatecmds.c b/src/backend/commands/aggregatecmds.c index d34c82c5ba..b36f09eb7b 100644 --- a/src/backend/commands/aggregatecmds.c +++ b/src/backend/commands/aggregatecmds.c @@ -52,8 +52,7 @@ * "parameters" is a list of DefElem representing the agg's definition clauses. */ ObjectAddress -DefineAggregate(List *name, List *args, bool oldstyle, List *parameters, - const char *queryString) +DefineAggregate(ParseState *pstate, List *name, List *args, bool oldstyle, List *parameters) { char *aggName; Oid aggNamespace; @@ -287,10 +286,10 @@ DefineAggregate(List *name, List *args, bool oldstyle, List *parameters, errmsg("basetype is redundant with aggregate input type specification"))); numArgs = list_length(args); - interpret_function_parameter_list(args, + interpret_function_parameter_list(pstate, + args, InvalidOid, true, /* is an aggregate */ - queryString, ¶meterTypes, &allParameterTypes, ¶meterModes, diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c index e4ebb654a6..0c75d16f36 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -38,7 +38,7 @@ * CREATE COLLATION */ ObjectAddress -DefineCollation(List *names, List *parameters) +DefineCollation(ParseState *pstate, List *names, List *parameters) { char *collName; Oid collNamespace; @@ -78,7 +78,8 @@ DefineCollation(List *names, List *parameters) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("collation attribute \"%s\" not recognized", - defel->defname))); + defel->defname), + parser_errposition(pstate, defel->location))); break; } diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 5947e72093..e393c0b39e 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -279,12 +279,12 @@ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"; /* non-export function prototypes */ -static CopyState BeginCopy(bool is_from, Relation rel, Node *raw_query, - const char *queryString, const Oid queryRelId, List *attnamelist, +static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, Node *raw_query, + const Oid queryRelId, List *attnamelist, List *options); static void EndCopy(CopyState cstate); static void ClosePipeToProgram(CopyState cstate); -static CopyState BeginCopyTo(Relation rel, Node *query, const char *queryString, +static CopyState BeginCopyTo(ParseState *pstate, Relation rel, Node *query, const Oid queryRelId, const char *filename, bool is_program, List *attnamelist, List *options); static void EndCopyTo(CopyState cstate); @@ -787,7 +787,7 @@ CopyLoadRawBuf(CopyState cstate) * the table or the specifically requested columns. */ Oid -DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed) +DoCopy(ParseState *pstate, const CopyStmt *stmt, uint64 *processed) { CopyState cstate; bool is_from = stmt->is_from; @@ -936,7 +936,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed) PreventCommandIfReadOnly("COPY FROM"); PreventCommandIfParallelMode("COPY FROM"); - cstate = BeginCopyFrom(rel, stmt->filename, stmt->is_program, + cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program, stmt->attlist, stmt->options); cstate->range_table = range_table; *processed = CopyFrom(cstate); /* copy from file to database */ @@ -944,7 +944,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed) } else { - cstate = BeginCopyTo(rel, query, queryString, relid, + cstate = BeginCopyTo(pstate, rel, query, relid, stmt->filename, stmt->is_program, stmt->attlist, stmt->options); *processed = DoCopyTo(cstate); /* copy from database to file */ @@ -980,7 +980,8 @@ DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed) * self-consistency of the options list. */ void -ProcessCopyOptions(CopyState cstate, +ProcessCopyOptions(ParseState *pstate, + CopyState cstate, bool is_from, List *options) { @@ -1005,7 +1006,8 @@ ProcessCopyOptions(CopyState cstate, if (format_specified) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); format_specified = true; if (strcmp(fmt, "text") == 0) /* default format */ ; @@ -1016,14 +1018,16 @@ ProcessCopyOptions(CopyState cstate, else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY format \"%s\" not recognized", fmt))); + errmsg("COPY format \"%s\" not recognized", fmt), + parser_errposition(pstate, defel->location))); } else if (strcmp(defel->defname, "oids") == 0) { if (cstate->oids) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cstate->oids = defGetBoolean(defel); } else if (strcmp(defel->defname, "freeze") == 0) @@ -1031,7 +1035,8 @@ ProcessCopyOptions(CopyState cstate, if (cstate->freeze) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cstate->freeze = defGetBoolean(defel); } else if (strcmp(defel->defname, "delimiter") == 0) @@ -1039,7 +1044,8 @@ ProcessCopyOptions(CopyState cstate, if (cstate->delim) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cstate->delim = defGetString(defel); } else if (strcmp(defel->defname, "null") == 0) @@ -1047,7 +1053,8 @@ ProcessCopyOptions(CopyState cstate, if (cstate->null_print) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cstate->null_print = defGetString(defel); } else if (strcmp(defel->defname, "header") == 0) @@ -1055,7 +1062,8 @@ ProcessCopyOptions(CopyState cstate, if (cstate->header_line) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cstate->header_line = defGetBoolean(defel); } else if (strcmp(defel->defname, "quote") == 0) @@ -1063,7 +1071,8 @@ ProcessCopyOptions(CopyState cstate, if (cstate->quote) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cstate->quote = defGetString(defel); } else if (strcmp(defel->defname, "escape") == 0) @@ -1071,7 +1080,8 @@ ProcessCopyOptions(CopyState cstate, if (cstate->escape) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cstate->escape = defGetString(defel); } else if (strcmp(defel->defname, "force_quote") == 0) @@ -1079,7 +1089,8 @@ ProcessCopyOptions(CopyState cstate, if (cstate->force_quote || cstate->force_quote_all) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); if (defel->arg && IsA(defel->arg, A_Star)) cstate->force_quote_all = true; else if (defel->arg && IsA(defel->arg, List)) @@ -1088,21 +1099,24 @@ ProcessCopyOptions(CopyState cstate, ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("argument to option \"%s\" must be a list of column names", - defel->defname))); + defel->defname), + parser_errposition(pstate, defel->location))); } else if (strcmp(defel->defname, "force_not_null") == 0) { if (cstate->force_notnull) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); if (defel->arg && IsA(defel->arg, List)) cstate->force_notnull = (List *) defel->arg; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("argument to option \"%s\" must be a list of column names", - defel->defname))); + defel->defname), + parser_errposition(pstate, defel->location))); } else if (strcmp(defel->defname, "force_null") == 0) { @@ -1116,7 +1130,8 @@ ProcessCopyOptions(CopyState cstate, ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("argument to option \"%s\" must be a list of column names", - defel->defname))); + defel->defname), + parser_errposition(pstate, defel->location))); } else if (strcmp(defel->defname, "convert_selectively") == 0) { @@ -1128,7 +1143,8 @@ ProcessCopyOptions(CopyState cstate, if (cstate->convert_selectively) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cstate->convert_selectively = true; if (defel->arg == NULL || IsA(defel->arg, List)) cstate->convert_select = (List *) defel->arg; @@ -1136,26 +1152,30 @@ ProcessCopyOptions(CopyState cstate, ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("argument to option \"%s\" must be a list of column names", - defel->defname))); + defel->defname), + parser_errposition(pstate, defel->location))); } else if (strcmp(defel->defname, "encoding") == 0) { if (cstate->file_encoding >= 0) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cstate->file_encoding = pg_char_to_encoding(defGetString(defel)); if (cstate->file_encoding < 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("argument to option \"%s\" must be a valid encoding name", - defel->defname))); + defel->defname), + parser_errposition(pstate, defel->location))); } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("option \"%s\" not recognized", - defel->defname))); + defel->defname), + parser_errposition(pstate, defel->location))); } /* @@ -1318,10 +1338,10 @@ ProcessCopyOptions(CopyState cstate, * NULL values as . */ static CopyState -BeginCopy(bool is_from, +BeginCopy(ParseState *pstate, + bool is_from, Relation rel, Node *raw_query, - const char *queryString, const Oid queryRelId, List *attnamelist, List *options) @@ -1345,7 +1365,7 @@ BeginCopy(bool is_from, oldcontext = MemoryContextSwitchTo(cstate->copycontext); /* Extract options from the statement node tree */ - ProcessCopyOptions(cstate, is_from, options); + ProcessCopyOptions(pstate, cstate, is_from, options); /* Process the source/target relation or query */ if (rel) @@ -1390,7 +1410,7 @@ BeginCopy(bool is_from, * DECLARE CURSOR and PREPARE.) XXX FIXME someday. */ rewritten = pg_analyze_and_rewrite((Node *) copyObject(raw_query), - queryString, NULL, 0); + pstate->p_sourcetext, NULL, 0); /* check that we got back something we can work with */ if (rewritten == NIL) @@ -1490,7 +1510,7 @@ BeginCopy(bool is_from, ((DR_copy *) dest)->cstate = cstate; /* Create a QueryDesc requesting no output */ - cstate->queryDesc = CreateQueryDesc(plan, queryString, + cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext, GetActiveSnapshot(), InvalidSnapshot, dest, NULL, 0); @@ -1678,9 +1698,9 @@ EndCopy(CopyState cstate) * Setup CopyState to read tuples from a table or a query for COPY TO. */ static CopyState -BeginCopyTo(Relation rel, +BeginCopyTo(ParseState *pstate, + Relation rel, Node *query, - const char *queryString, const Oid queryRelId, const char *filename, bool is_program, @@ -1723,7 +1743,7 @@ BeginCopyTo(Relation rel, RelationGetRelationName(rel)))); } - cstate = BeginCopy(false, rel, query, queryString, queryRelId, attnamelist, + cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist, options); oldcontext = MemoryContextSwitchTo(cstate->copycontext); @@ -2645,7 +2665,8 @@ CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, * Returns a CopyState, to be passed to NextCopyFrom and related functions. */ CopyState -BeginCopyFrom(Relation rel, +BeginCopyFrom(ParseState *pstate, + Relation rel, const char *filename, bool is_program, List *attnamelist, @@ -2666,7 +2687,7 @@ BeginCopyFrom(Relation rel, MemoryContext oldcontext; bool volatile_defexprs; - cstate = BeginCopy(true, rel, NULL, NULL, InvalidOid, attnamelist, options); + cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options); oldcontext = MemoryContextSwitchTo(cstate->copycontext); /* Initialize state variables */ diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index c1c0223770..ef486593c0 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -96,7 +96,7 @@ static int errdetail_busy_db(int notherbackends, int npreparedxacts); * CREATE DATABASE */ Oid -createdb(const CreatedbStmt *stmt) +createdb(ParseState *pstate, const CreatedbStmt *stmt) { HeapScanDesc scan; Relation rel; @@ -152,7 +152,8 @@ createdb(const CreatedbStmt *stmt) if (dtablespacename) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dtablespacename = defel; } else if (strcmp(defel->defname, "owner") == 0) @@ -160,7 +161,8 @@ createdb(const CreatedbStmt *stmt) if (downer) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); downer = defel; } else if (strcmp(defel->defname, "template") == 0) @@ -168,7 +170,8 @@ createdb(const CreatedbStmt *stmt) if (dtemplate) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dtemplate = defel; } else if (strcmp(defel->defname, "encoding") == 0) @@ -176,7 +179,8 @@ createdb(const CreatedbStmt *stmt) if (dencoding) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dencoding = defel; } else if (strcmp(defel->defname, "lc_collate") == 0) @@ -184,7 +188,8 @@ createdb(const CreatedbStmt *stmt) if (dcollate) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dcollate = defel; } else if (strcmp(defel->defname, "lc_ctype") == 0) @@ -192,7 +197,8 @@ createdb(const CreatedbStmt *stmt) if (dctype) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dctype = defel; } else if (strcmp(defel->defname, "is_template") == 0) @@ -200,7 +206,8 @@ createdb(const CreatedbStmt *stmt) if (distemplate) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); distemplate = defel; } else if (strcmp(defel->defname, "allow_connections") == 0) @@ -208,7 +215,8 @@ createdb(const CreatedbStmt *stmt) if (dallowconnections) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dallowconnections = defel; } else if (strcmp(defel->defname, "connection_limit") == 0) @@ -216,7 +224,8 @@ createdb(const CreatedbStmt *stmt) if (dconnlimit) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dconnlimit = defel; } else if (strcmp(defel->defname, "location") == 0) @@ -224,12 +233,14 @@ createdb(const CreatedbStmt *stmt) ereport(WARNING, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("LOCATION is not supported anymore"), - errhint("Consider using tablespaces instead."))); + errhint("Consider using tablespaces instead."), + parser_errposition(pstate, defel->location))); } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("option \"%s\" not recognized", defel->defname))); + errmsg("option \"%s\" not recognized", defel->defname), + parser_errposition(pstate, defel->location))); } if (downer && downer->arg) @@ -249,7 +260,8 @@ createdb(const CreatedbStmt *stmt) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("%d is not a valid encoding code", - encoding))); + encoding), + parser_errposition(pstate, dencoding->location))); } else { @@ -259,7 +271,8 @@ createdb(const CreatedbStmt *stmt) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("%s is not a valid encoding name", - encoding_name))); + encoding_name), + parser_errposition(pstate, dencoding->location))); } } if (dcollate && dcollate->arg) @@ -1364,7 +1377,7 @@ movedb_failure_callback(int code, Datum arg) * ALTER DATABASE name ... */ Oid -AlterDatabase(AlterDatabaseStmt *stmt, bool isTopLevel) +AlterDatabase(ParseState *pstate, AlterDatabaseStmt *stmt, bool isTopLevel) { Relation rel; Oid dboid; @@ -1394,7 +1407,8 @@ AlterDatabase(AlterDatabaseStmt *stmt, bool isTopLevel) if (distemplate) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); distemplate = defel; } else if (strcmp(defel->defname, "allow_connections") == 0) @@ -1402,7 +1416,8 @@ AlterDatabase(AlterDatabaseStmt *stmt, bool isTopLevel) if (dallowconnections) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dallowconnections = defel; } else if (strcmp(defel->defname, "connection_limit") == 0) @@ -1410,7 +1425,8 @@ AlterDatabase(AlterDatabaseStmt *stmt, bool isTopLevel) if (dconnlimit) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dconnlimit = defel; } else if (strcmp(defel->defname, "tablespace") == 0) @@ -1418,13 +1434,15 @@ AlterDatabase(AlterDatabaseStmt *stmt, bool isTopLevel) if (dtablespace) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dtablespace = defel; } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("option \"%s\" not recognized", defel->defname))); + errmsg("option \"%s\" not recognized", defel->defname), + parser_errposition(pstate, defel->location))); } if (dtablespace) @@ -1438,7 +1456,8 @@ AlterDatabase(AlterDatabaseStmt *stmt, bool isTopLevel) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("option \"%s\" cannot be specified with other options", - dtablespace->defname))); + dtablespace->defname), + parser_errposition(pstate, dtablespace->location))); /* this case isn't allowed within a transaction block */ PreventTransactionChain(isTopLevel, "ALTER DATABASE SET TABLESPACE"); movedb(stmt->dbname, defGetString(dtablespace)); diff --git a/src/backend/commands/define.c b/src/backend/commands/define.c index ece803ec4b..533926693a 100644 --- a/src/backend/commands/define.c +++ b/src/backend/commands/define.c @@ -319,12 +319,3 @@ defGetTypeLength(DefElem *def) def->defname, defGetString(def)))); return 0; /* keep compiler quiet */ } - -/* - * Create a DefElem setting "oids" to the specified value. - */ -DefElem * -defWithOids(bool value) -{ - return makeDefElem("oids", (Node *) makeInteger(value)); -} diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 82ba58ef71..124743318d 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -140,7 +140,7 @@ static void escape_yaml(StringInfo buf, const char *str); * execute an EXPLAIN command */ void -ExplainQuery(ExplainStmt *stmt, const char *queryString, +ExplainQuery(ParseState *pstate, ExplainStmt *stmt, const char *queryString, ParamListInfo params, DestReceiver *dest) { ExplainState *es = NewExplainState(); @@ -183,13 +183,15 @@ ExplainQuery(ExplainStmt *stmt, const char *queryString, ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized value for EXPLAIN option \"%s\": \"%s\"", - opt->defname, p))); + opt->defname, p), + parser_errposition(pstate, opt->location))); } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized EXPLAIN option \"%s\"", - opt->defname))); + opt->defname), + parser_errposition(pstate, opt->location))); } if (es->buffers && !es->analyze) diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c index fa861e670b..df49a78e2f 100644 --- a/src/backend/commands/extension.c +++ b/src/backend/commands/extension.c @@ -1175,7 +1175,7 @@ find_update_path(List *evi_list, * installed, allowing us to error out if we recurse to one of those. */ static ObjectAddress -CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) +CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *parents) { DefElem *d_schema = NULL; DefElem *d_new_version = NULL; @@ -1215,7 +1215,8 @@ CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) if (d_schema) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); d_schema = defel; } else if (strcmp(defel->defname, "new_version") == 0) @@ -1223,7 +1224,8 @@ CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) if (d_new_version) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); d_new_version = defel; } else if (strcmp(defel->defname, "old_version") == 0) @@ -1231,7 +1233,8 @@ CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) if (d_old_version) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); d_old_version = defel; } else if (strcmp(defel->defname, "cascade") == 0) @@ -1239,7 +1242,8 @@ CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) if (d_cascade) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); d_cascade = defel; cascade = defGetBoolean(d_cascade); } @@ -1458,7 +1462,7 @@ CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) lappend(list_copy(parents), stmt->extname); /* Create the required extension. */ - addr = CreateExtensionInternal(ces, cascade_parents); + addr = CreateExtensionInternal(pstate, ces, cascade_parents); /* Get its newly-assigned OID. */ reqext = addr.objectId; @@ -1515,7 +1519,7 @@ CreateExtensionInternal(CreateExtensionStmt *stmt, List *parents) * CREATE EXTENSION */ ObjectAddress -CreateExtension(CreateExtensionStmt *stmt) +CreateExtension(ParseState *pstate, CreateExtensionStmt *stmt) { /* Check extension name validity before any filesystem access */ check_valid_extension_name(stmt->extname); @@ -1553,7 +1557,7 @@ CreateExtension(CreateExtensionStmt *stmt) errmsg("nested CREATE EXTENSION is not supported"))); /* Finally create the extension. */ - return CreateExtensionInternal(stmt, NIL); + return CreateExtensionInternal(pstate, stmt, NIL); } /* @@ -2671,7 +2675,7 @@ AlterExtensionNamespace(List *names, const char *newschema, Oid *oldschema) * Execute ALTER EXTENSION UPDATE */ ObjectAddress -ExecAlterExtensionStmt(AlterExtensionStmt *stmt) +ExecAlterExtensionStmt(ParseState *pstate, AlterExtensionStmt *stmt) { DefElem *d_new_version = NULL; char *versionName; @@ -2757,7 +2761,8 @@ ExecAlterExtensionStmt(AlterExtensionStmt *stmt) if (d_new_version) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); d_new_version = defel; } else diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index 748c8f75d4..becafc3045 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -167,7 +167,6 @@ compute_return_type(TypeName *returnType, Oid languageOid, * parameters: list of FunctionParameter structs * languageOid: OID of function language (InvalidOid if it's CREATE AGGREGATE) * is_aggregate: needed only to determine error handling - * queryString: likewise, needed only for error handling * * Results are stored into output parameters. parameterTypes must always * be created, but the other arrays are set to NULL if not needed. @@ -177,10 +176,10 @@ compute_return_type(TypeName *returnType, Oid languageOid, * else it is set to the OID of the implied result type. */ void -interpret_function_parameter_list(List *parameters, +interpret_function_parameter_list(ParseState *pstate, + List *parameters, Oid languageOid, bool is_aggregate, - const char *queryString, oidvector **parameterTypes, ArrayType **allParameterTypes, ArrayType **parameterModes, @@ -201,7 +200,6 @@ interpret_function_parameter_list(List *parameters, bool have_defaults = false; ListCell *x; int i; - ParseState *pstate; *variadicArgType = InvalidOid; /* default result */ *requiredResultType = InvalidOid; /* default result */ @@ -212,10 +210,6 @@ interpret_function_parameter_list(List *parameters, paramNames = (Datum *) palloc0(parameterCount * sizeof(Datum)); *parameterDefaults = NIL; - /* may need a pstate for parse analysis of default exprs */ - pstate = make_parsestate(NULL); - pstate->p_sourcetext = queryString; - /* Scan the list and extract data into work arrays */ i = 0; foreach(x, parameters) @@ -413,8 +407,6 @@ interpret_function_parameter_list(List *parameters, i++; } - free_parsestate(pstate); - /* Now construct the proper outputs as needed */ *parameterTypes = buildoidvector(inTypes, inCount); @@ -458,7 +450,8 @@ interpret_function_parameter_list(List *parameters, * SET parameters though --- if you're redundant, the last one wins.) */ static bool -compute_common_attribute(DefElem *defel, +compute_common_attribute(ParseState *pstate, + DefElem *defel, DefElem **volatility_item, DefElem **strict_item, DefElem **security_item, @@ -530,7 +523,8 @@ compute_common_attribute(DefElem *defel, duplicate_error: ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); return false; /* keep compiler quiet */ } @@ -609,7 +603,8 @@ update_proconfig_value(ArrayType *a, List *set_items) * attributes. */ static void -compute_attributes_sql_style(List *options, +compute_attributes_sql_style(ParseState *pstate, + List *options, List **as, char **language, Node **transform, @@ -646,7 +641,8 @@ compute_attributes_sql_style(List *options, if (as_item) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); as_item = defel; } else if (strcmp(defel->defname, "language") == 0) @@ -654,7 +650,8 @@ compute_attributes_sql_style(List *options, if (language_item) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); language_item = defel; } else if (strcmp(defel->defname, "transform") == 0) @@ -662,7 +659,8 @@ compute_attributes_sql_style(List *options, if (transform_item) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); transform_item = defel; } else if (strcmp(defel->defname, "window") == 0) @@ -670,10 +668,12 @@ compute_attributes_sql_style(List *options, if (windowfunc_item) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); windowfunc_item = defel; } - else if (compute_common_attribute(defel, + else if (compute_common_attribute(pstate, + defel, &volatility_item, &strict_item, &security_item, @@ -763,7 +763,7 @@ compute_attributes_sql_style(List *options, *------------ */ static void -compute_attributes_with_style(List *parameters, bool *isStrict_p, char *volatility_p) +compute_attributes_with_style(ParseState *pstate, List *parameters, bool *isStrict_p, char *volatility_p) { ListCell *pl; @@ -783,7 +783,8 @@ compute_attributes_with_style(List *parameters, bool *isStrict_p, char *volatili ereport(WARNING, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized function attribute \"%s\" ignored", - param->defname))); + param->defname), + parser_errposition(pstate, param->location))); } } @@ -858,7 +859,7 @@ interpret_AS_clause(Oid languageOid, const char *languageName, * Execute a CREATE FUNCTION utility statement. */ ObjectAddress -CreateFunction(CreateFunctionStmt *stmt, const char *queryString) +CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) { char *probin_str; char *prosrc_str; @@ -915,7 +916,8 @@ CreateFunction(CreateFunctionStmt *stmt, const char *queryString) parallel = PROPARALLEL_UNSAFE; /* override attributes from explicit list */ - compute_attributes_sql_style(stmt->options, + compute_attributes_sql_style(pstate, + stmt->options, &as_clause, &language, &transformDefElem, &isWindowFunc, &volatility, &isStrict, &security, &isLeakProof, @@ -987,10 +989,10 @@ CreateFunction(CreateFunctionStmt *stmt, const char *queryString) * Convert remaining parameters of CREATE to form wanted by * ProcedureCreate. */ - interpret_function_parameter_list(stmt->parameters, + interpret_function_parameter_list(pstate, + stmt->parameters, languageOid, false, /* not an aggregate */ - queryString, ¶meterTypes, &allParameterTypes, ¶meterModes, @@ -1045,7 +1047,7 @@ CreateFunction(CreateFunctionStmt *stmt, const char *queryString) trftypes = NULL; } - compute_attributes_with_style(stmt->withClause, &isStrict, &volatility); + compute_attributes_with_style(pstate, stmt->withClause, &isStrict, &volatility); interpret_AS_clause(languageOid, language, funcname, as_clause, &prosrc_str, &probin_str); @@ -1163,7 +1165,7 @@ RemoveFunctionById(Oid funcOid) * ALTER framework). */ ObjectAddress -AlterFunction(AlterFunctionStmt *stmt) +AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) { HeapTuple tup; Oid funcOid; @@ -1208,7 +1210,8 @@ AlterFunction(AlterFunctionStmt *stmt) { DefElem *defel = (DefElem *) lfirst(l); - if (compute_common_attribute(defel, + if (compute_common_attribute(pstate, + defel, &volatility_item, &strict_item, &security_def_item, diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c index c98f981111..fc3a8eebce 100644 --- a/src/backend/commands/sequence.c +++ b/src/backend/commands/sequence.c @@ -94,7 +94,7 @@ static void create_seq_hashtable(void); static void init_sequence(Oid relid, SeqTable *p_elm, Relation *p_rel); static Form_pg_sequence read_seq_tuple(SeqTable elm, Relation rel, Buffer *buf, HeapTuple seqtuple); -static void init_params(List *options, bool isInit, +static void init_params(ParseState *pstate, List *options, bool isInit, Form_pg_sequence new, List **owned_by); static void do_setval(Oid relid, int64 next, bool iscalled); static void process_owned_by(Relation seqrel, List *owned_by); @@ -105,7 +105,7 @@ static void process_owned_by(Relation seqrel, List *owned_by); * Creates a new sequence relation */ ObjectAddress -DefineSequence(CreateSeqStmt *seq) +DefineSequence(ParseState *pstate, CreateSeqStmt *seq) { FormData_pg_sequence new; List *owned_by; @@ -145,7 +145,7 @@ DefineSequence(CreateSeqStmt *seq) } /* Check and set all option values */ - init_params(seq->options, true, &new, &owned_by); + init_params(pstate, seq->options, true, &new, &owned_by); /* * Create relation (and fill value[] and null[] for the tuple) @@ -404,7 +404,7 @@ fill_seq_with_data(Relation rel, HeapTuple tuple) * Modify the definition of a sequence relation */ ObjectAddress -AlterSequence(AlterSeqStmt *stmt) +AlterSequence(ParseState *pstate, AlterSeqStmt *stmt) { Oid relid; SeqTable elm; @@ -440,7 +440,7 @@ AlterSequence(AlterSeqStmt *stmt) memcpy(&new, seq, sizeof(FormData_pg_sequence)); /* Check and set new values */ - init_params(stmt->options, false, &new, &owned_by); + init_params(pstate, stmt->options, false, &new, &owned_by); /* Clear local cache so that we don't think we have cached numbers */ /* Note that we do not change the currval() state */ @@ -1163,7 +1163,7 @@ read_seq_tuple(SeqTable elm, Relation rel, Buffer *buf, HeapTuple seqtuple) * otherwise, do not change existing options that aren't explicitly overridden. */ static void -init_params(List *options, bool isInit, +init_params(ParseState *pstate, List *options, bool isInit, Form_pg_sequence new, List **owned_by) { DefElem *start_value = NULL; @@ -1186,7 +1186,8 @@ init_params(List *options, bool isInit, if (increment_by) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); increment_by = defel; } else if (strcmp(defel->defname, "start") == 0) @@ -1194,7 +1195,8 @@ init_params(List *options, bool isInit, if (start_value) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); start_value = defel; } else if (strcmp(defel->defname, "restart") == 0) @@ -1202,7 +1204,8 @@ init_params(List *options, bool isInit, if (restart_value) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); restart_value = defel; } else if (strcmp(defel->defname, "maxvalue") == 0) @@ -1210,7 +1213,8 @@ init_params(List *options, bool isInit, if (max_value) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); max_value = defel; } else if (strcmp(defel->defname, "minvalue") == 0) @@ -1218,7 +1222,8 @@ init_params(List *options, bool isInit, if (min_value) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); min_value = defel; } else if (strcmp(defel->defname, "cache") == 0) @@ -1226,7 +1231,8 @@ init_params(List *options, bool isInit, if (cache_value) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); cache_value = defel; } else if (strcmp(defel->defname, "cycle") == 0) @@ -1234,7 +1240,8 @@ init_params(List *options, bool isInit, if (is_cycled) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); is_cycled = defel; } else if (strcmp(defel->defname, "owned_by") == 0) @@ -1242,7 +1249,8 @@ init_params(List *options, bool isInit, if (*owned_by) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); *owned_by = defGetQualifiedName(defel); } else diff --git a/src/backend/commands/tsearchcmds.c b/src/backend/commands/tsearchcmds.c index 69c038c52b..b24011371c 100644 --- a/src/backend/commands/tsearchcmds.c +++ b/src/backend/commands/tsearchcmds.c @@ -1700,7 +1700,7 @@ deserialize_deflist(Datum txt) *wsptr++ = '\0'; result = lappend(result, makeDefElem(pstrdup(workspace), - (Node *) makeString(pstrdup(startvalue)))); + (Node *) makeString(pstrdup(startvalue)), -1)); state = CS_WAITKEY; } } @@ -1732,7 +1732,7 @@ deserialize_deflist(Datum txt) *wsptr++ = '\0'; result = lappend(result, makeDefElem(pstrdup(workspace), - (Node *) makeString(pstrdup(startvalue)))); + (Node *) makeString(pstrdup(startvalue)), -1)); state = CS_WAITKEY; } } @@ -1747,7 +1747,7 @@ deserialize_deflist(Datum txt) *wsptr++ = '\0'; result = lappend(result, makeDefElem(pstrdup(workspace), - (Node *) makeString(pstrdup(startvalue)))); + (Node *) makeString(pstrdup(startvalue)), -1)); state = CS_WAITKEY; } else @@ -1766,7 +1766,7 @@ deserialize_deflist(Datum txt) *wsptr++ = '\0'; result = lappend(result, makeDefElem(pstrdup(workspace), - (Node *) makeString(pstrdup(startvalue)))); + (Node *) makeString(pstrdup(startvalue)), -1)); } else if (state != CS_WAITKEY) ereport(ERROR, diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c index 8e7be78f65..6cc7106467 100644 --- a/src/backend/commands/typecmds.c +++ b/src/backend/commands/typecmds.c @@ -111,7 +111,7 @@ static char *domainAddConstraint(Oid domainOid, Oid domainNamespace, * Registers a new base type. */ ObjectAddress -DefineType(List *names, List *parameters) +DefineType(ParseState *pstate, List *names, List *parameters) { char *typeName; Oid typeNamespace; @@ -286,13 +286,15 @@ DefineType(List *names, List *parameters) ereport(WARNING, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("type attribute \"%s\" not recognized", - defel->defname))); + defel->defname), + parser_errposition(pstate, defel->location))); continue; } if (*defelp != NULL) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); *defelp = defel; } diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index 821dce3ce7..4027c89b14 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -69,7 +69,7 @@ have_createrole_privilege(void) * CREATE ROLE */ Oid -CreateRole(CreateRoleStmt *stmt) +CreateRole(ParseState *pstate, CreateRoleStmt *stmt) { Relation pg_authid_rel; TupleDesc pg_authid_dsc; @@ -136,7 +136,8 @@ CreateRole(CreateRoleStmt *stmt) if (dpassword) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dpassword = defel; if (strcmp(defel->defname, "encryptedPassword") == 0) encrypt_password = true; @@ -153,7 +154,8 @@ CreateRole(CreateRoleStmt *stmt) if (dissuper) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dissuper = defel; } else if (strcmp(defel->defname, "inherit") == 0) @@ -161,7 +163,8 @@ CreateRole(CreateRoleStmt *stmt) if (dinherit) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dinherit = defel; } else if (strcmp(defel->defname, "createrole") == 0) @@ -169,7 +172,8 @@ CreateRole(CreateRoleStmt *stmt) if (dcreaterole) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dcreaterole = defel; } else if (strcmp(defel->defname, "createdb") == 0) @@ -177,7 +181,8 @@ CreateRole(CreateRoleStmt *stmt) if (dcreatedb) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dcreatedb = defel; } else if (strcmp(defel->defname, "canlogin") == 0) @@ -185,7 +190,8 @@ CreateRole(CreateRoleStmt *stmt) if (dcanlogin) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dcanlogin = defel; } else if (strcmp(defel->defname, "isreplication") == 0) @@ -193,7 +199,8 @@ CreateRole(CreateRoleStmt *stmt) if (disreplication) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); disreplication = defel; } else if (strcmp(defel->defname, "connectionlimit") == 0) @@ -201,7 +208,8 @@ CreateRole(CreateRoleStmt *stmt) if (dconnlimit) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dconnlimit = defel; } else if (strcmp(defel->defname, "addroleto") == 0) @@ -209,7 +217,8 @@ CreateRole(CreateRoleStmt *stmt) if (daddroleto) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); daddroleto = defel; } else if (strcmp(defel->defname, "rolemembers") == 0) @@ -217,7 +226,8 @@ CreateRole(CreateRoleStmt *stmt) if (drolemembers) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); drolemembers = defel; } else if (strcmp(defel->defname, "adminmembers") == 0) @@ -225,7 +235,8 @@ CreateRole(CreateRoleStmt *stmt) if (dadminmembers) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dadminmembers = defel; } else if (strcmp(defel->defname, "validUntil") == 0) @@ -233,7 +244,8 @@ CreateRole(CreateRoleStmt *stmt) if (dvalidUntil) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dvalidUntil = defel; } else if (strcmp(defel->defname, "bypassrls") == 0) @@ -241,7 +253,8 @@ CreateRole(CreateRoleStmt *stmt) if (dbypassRLS) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); dbypassRLS = defel; } else diff --git a/src/backend/commands/view.c b/src/backend/commands/view.c index 085bf32320..325a81096f 100644 --- a/src/backend/commands/view.c +++ b/src/backend/commands/view.c @@ -434,11 +434,11 @@ DefineView(ViewStmt *stmt, const char *queryString) if (stmt->withCheckOption == LOCAL_CHECK_OPTION) stmt->options = lappend(stmt->options, makeDefElem("check_option", - (Node *) makeString("local"))); + (Node *) makeString("local"), -1)); else if (stmt->withCheckOption == CASCADED_CHECK_OPTION) stmt->options = lappend(stmt->options, makeDefElem("check_option", - (Node *) makeString("cascaded"))); + (Node *) makeString("cascaded"), -1)); /* * Check that the view is auto-updatable if WITH CHECK OPTION was diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 1877fb45e5..be2207e318 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2676,6 +2676,7 @@ _copyDefElem(const DefElem *from) COPY_STRING_FIELD(defname); COPY_NODE_FIELD(arg); COPY_SCALAR_FIELD(defaction); + COPY_LOCATION_FIELD(location); return newnode; } diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 448e1a9d55..c4ec4077a6 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2424,6 +2424,7 @@ _equalDefElem(const DefElem *a, const DefElem *b) COMPARE_STRING_FIELD(defname); COMPARE_NODE_FIELD(arg); COMPARE_SCALAR_FIELD(defaction); + COMPARE_LOCATION_FIELD(location); return true; } @@ -2434,6 +2435,7 @@ _equalLockingClause(const LockingClause *a, const LockingClause *b) COMPARE_NODE_FIELD(lockedRels); COMPARE_SCALAR_FIELD(strength); COMPARE_SCALAR_FIELD(waitPolicy); + COMPARE_LOCATION_FIELD(location); return true; } diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c index d72a85ee1b..20e2dbda79 100644 --- a/src/backend/nodes/makefuncs.c +++ b/src/backend/nodes/makefuncs.c @@ -540,7 +540,7 @@ makeFuncExpr(Oid funcid, Oid rettype, List *args, * and no special action. */ DefElem * -makeDefElem(char *name, Node *arg) +makeDefElem(char *name, Node *arg, int location) { DefElem *res = makeNode(DefElem); @@ -548,6 +548,7 @@ makeDefElem(char *name, Node *arg) res->defname = name; res->arg = arg; res->defaction = DEFELEM_UNSPEC; + res->location = location; return res; } @@ -558,7 +559,7 @@ makeDefElem(char *name, Node *arg) */ DefElem * makeDefElemExtended(char *nameSpace, char *name, Node *arg, - DefElemAction defaction) + DefElemAction defaction, int location) { DefElem *res = makeNode(DefElem); @@ -566,6 +567,7 @@ makeDefElemExtended(char *nameSpace, char *name, Node *arg, res->defname = name; res->arg = arg; res->defaction = defaction; + res->location = location; return res; } diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 29b7712584..90fecb1338 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2542,6 +2542,7 @@ _outDefElem(StringInfo str, const DefElem *node) WRITE_STRING_FIELD(defname); WRITE_NODE_FIELD(arg); WRITE_ENUM_FIELD(defaction, DefElemAction); + WRITE_LOCATION_FIELD(location); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 6f9a81e3e3..894a48fb4f 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1388,6 +1388,7 @@ _readDefElem(void) READ_STRING_FIELD(defname); READ_NODE_FIELD(arg); READ_ENUM_FIELD(defaction, DefElemAction); + READ_LOCATION_FIELD(location); READ_DONE(); } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index cb5cfc480c..b69a77a588 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -928,38 +928,38 @@ AlterOptRoleElem: PASSWORD Sconst { $$ = makeDefElem("password", - (Node *)makeString($2)); + (Node *)makeString($2), @1); } | PASSWORD NULL_P { - $$ = makeDefElem("password", NULL); + $$ = makeDefElem("password", NULL, @1); } | ENCRYPTED PASSWORD Sconst { $$ = makeDefElem("encryptedPassword", - (Node *)makeString($3)); + (Node *)makeString($3), @1); } | UNENCRYPTED PASSWORD Sconst { $$ = makeDefElem("unencryptedPassword", - (Node *)makeString($3)); + (Node *)makeString($3), @1); } | INHERIT { - $$ = makeDefElem("inherit", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("inherit", (Node *)makeInteger(TRUE), @1); } | CONNECTION LIMIT SignedIconst { - $$ = makeDefElem("connectionlimit", (Node *)makeInteger($3)); + $$ = makeDefElem("connectionlimit", (Node *)makeInteger($3), @1); } | VALID UNTIL Sconst { - $$ = makeDefElem("validUntil", (Node *)makeString($3)); + $$ = makeDefElem("validUntil", (Node *)makeString($3), @1); } /* Supported but not documented for roles, for use by ALTER GROUP. */ | USER role_list { - $$ = makeDefElem("rolemembers", (Node *)$2); + $$ = makeDefElem("rolemembers", (Node *)$2, @1); } | IDENT { @@ -969,36 +969,36 @@ AlterOptRoleElem: * size of the main parser. */ if (strcmp($1, "superuser") == 0) - $$ = makeDefElem("superuser", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("superuser", (Node *)makeInteger(TRUE), @1); else if (strcmp($1, "nosuperuser") == 0) - $$ = makeDefElem("superuser", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("superuser", (Node *)makeInteger(FALSE), @1); else if (strcmp($1, "createrole") == 0) - $$ = makeDefElem("createrole", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("createrole", (Node *)makeInteger(TRUE), @1); else if (strcmp($1, "nocreaterole") == 0) - $$ = makeDefElem("createrole", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("createrole", (Node *)makeInteger(FALSE), @1); else if (strcmp($1, "replication") == 0) - $$ = makeDefElem("isreplication", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("isreplication", (Node *)makeInteger(TRUE), @1); else if (strcmp($1, "noreplication") == 0) - $$ = makeDefElem("isreplication", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("isreplication", (Node *)makeInteger(FALSE), @1); else if (strcmp($1, "createdb") == 0) - $$ = makeDefElem("createdb", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("createdb", (Node *)makeInteger(TRUE), @1); else if (strcmp($1, "nocreatedb") == 0) - $$ = makeDefElem("createdb", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("createdb", (Node *)makeInteger(FALSE), @1); else if (strcmp($1, "login") == 0) - $$ = makeDefElem("canlogin", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("canlogin", (Node *)makeInteger(TRUE), @1); else if (strcmp($1, "nologin") == 0) - $$ = makeDefElem("canlogin", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("canlogin", (Node *)makeInteger(FALSE), @1); else if (strcmp($1, "bypassrls") == 0) - $$ = makeDefElem("bypassrls", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("bypassrls", (Node *)makeInteger(TRUE), @1); else if (strcmp($1, "nobypassrls") == 0) - $$ = makeDefElem("bypassrls", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("bypassrls", (Node *)makeInteger(FALSE), @1); else if (strcmp($1, "noinherit") == 0) { /* * Note that INHERIT is a keyword, so it's handled by main parser, but * NOINHERIT is handled here. */ - $$ = makeDefElem("inherit", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("inherit", (Node *)makeInteger(FALSE), @1); } else ereport(ERROR, @@ -1013,23 +1013,23 @@ CreateOptRoleElem: /* The following are not supported by ALTER ROLE/USER/GROUP */ | SYSID Iconst { - $$ = makeDefElem("sysid", (Node *)makeInteger($2)); + $$ = makeDefElem("sysid", (Node *)makeInteger($2), @1); } | ADMIN role_list { - $$ = makeDefElem("adminmembers", (Node *)$2); + $$ = makeDefElem("adminmembers", (Node *)$2, @1); } | ROLE role_list { - $$ = makeDefElem("rolemembers", (Node *)$2); + $$ = makeDefElem("rolemembers", (Node *)$2, @1); } | IN_P ROLE role_list { - $$ = makeDefElem("addroleto", (Node *)$3); + $$ = makeDefElem("addroleto", (Node *)$3, @1); } | IN_P GROUP_P role_list { - $$ = makeDefElem("addroleto", (Node *)$3); + $$ = makeDefElem("addroleto", (Node *)$3, @1); } ; @@ -1206,7 +1206,7 @@ AlterGroupStmt: n->role = $3; n->action = $4; n->options = list_make1(makeDefElem("rolemembers", - (Node *)$6)); + (Node *)$6, @6)); $$ = (Node *)n; } ; @@ -2446,20 +2446,20 @@ reloption_list: reloption_elem: ColLabel '=' def_arg { - $$ = makeDefElem($1, (Node *) $3); + $$ = makeDefElem($1, (Node *) $3, @1); } | ColLabel { - $$ = makeDefElem($1, NULL); + $$ = makeDefElem($1, NULL, @1); } | ColLabel '.' ColLabel '=' def_arg { $$ = makeDefElemExtended($1, $3, (Node *) $5, - DEFELEM_UNSPEC); + DEFELEM_UNSPEC, @1); } | ColLabel '.' ColLabel { - $$ = makeDefElemExtended($1, $3, NULL, DEFELEM_UNSPEC); + $$ = makeDefElemExtended($1, $3, NULL, DEFELEM_UNSPEC, @1); } ; @@ -2669,59 +2669,59 @@ copy_opt_list: copy_opt_item: BINARY { - $$ = makeDefElem("format", (Node *)makeString("binary")); + $$ = makeDefElem("format", (Node *)makeString("binary"), @1); } | OIDS { - $$ = makeDefElem("oids", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("oids", (Node *)makeInteger(TRUE), @1); } | FREEZE { - $$ = makeDefElem("freeze", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("freeze", (Node *)makeInteger(TRUE), @1); } | DELIMITER opt_as Sconst { - $$ = makeDefElem("delimiter", (Node *)makeString($3)); + $$ = makeDefElem("delimiter", (Node *)makeString($3), @1); } | NULL_P opt_as Sconst { - $$ = makeDefElem("null", (Node *)makeString($3)); + $$ = makeDefElem("null", (Node *)makeString($3), @1); } | CSV { - $$ = makeDefElem("format", (Node *)makeString("csv")); + $$ = makeDefElem("format", (Node *)makeString("csv"), @1); } | HEADER_P { - $$ = makeDefElem("header", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("header", (Node *)makeInteger(TRUE), @1); } | QUOTE opt_as Sconst { - $$ = makeDefElem("quote", (Node *)makeString($3)); + $$ = makeDefElem("quote", (Node *)makeString($3), @1); } | ESCAPE opt_as Sconst { - $$ = makeDefElem("escape", (Node *)makeString($3)); + $$ = makeDefElem("escape", (Node *)makeString($3), @1); } | FORCE QUOTE columnList { - $$ = makeDefElem("force_quote", (Node *)$3); + $$ = makeDefElem("force_quote", (Node *)$3, @1); } | FORCE QUOTE '*' { - $$ = makeDefElem("force_quote", (Node *)makeNode(A_Star)); + $$ = makeDefElem("force_quote", (Node *)makeNode(A_Star), @1); } | FORCE NOT NULL_P columnList { - $$ = makeDefElem("force_not_null", (Node *)$4); + $$ = makeDefElem("force_not_null", (Node *)$4, @1); } | FORCE NULL_P columnList { - $$ = makeDefElem("force_null", (Node *)$3); + $$ = makeDefElem("force_null", (Node *)$3, @1); } | ENCODING Sconst { - $$ = makeDefElem("encoding", (Node *)makeString($2)); + $$ = makeDefElem("encoding", (Node *)makeString($2), @1); } ; @@ -2730,7 +2730,7 @@ copy_opt_item: opt_binary: BINARY { - $$ = makeDefElem("format", (Node *)makeString("binary")); + $$ = makeDefElem("format", (Node *)makeString("binary"), @1); } | /*EMPTY*/ { $$ = NULL; } ; @@ -2738,7 +2738,7 @@ opt_binary: opt_oids: WITH OIDS { - $$ = makeDefElem("oids", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("oids", (Node *)makeInteger(TRUE), @1); } | /*EMPTY*/ { $$ = NULL; } ; @@ -2746,7 +2746,7 @@ opt_oids: copy_delimiter: opt_using DELIMITERS Sconst { - $$ = makeDefElem("delimiter", (Node *)makeString($3)); + $$ = makeDefElem("delimiter", (Node *)makeString($3), @2); } | /*EMPTY*/ { $$ = NULL; } ; @@ -2771,7 +2771,7 @@ copy_generic_opt_list: copy_generic_opt_elem: ColLabel copy_generic_opt_arg { - $$ = makeDefElem($1, $2); + $$ = makeDefElem($1, $2, @1); } ; @@ -3418,8 +3418,8 @@ OptInherit: INHERITS '(' qualified_name_list ')' { $$ = $3; } /* WITH (options) is preferred, WITH OIDS and WITHOUT OIDS are legacy forms */ OptWith: WITH reloptions { $$ = $2; } - | WITH OIDS { $$ = list_make1(defWithOids(true)); } - | WITHOUT OIDS { $$ = list_make1(defWithOids(false)); } + | WITH OIDS { $$ = list_make1(makeDefElem("oids", (Node *) makeInteger(true), @1)); } + | WITHOUT OIDS { $$ = list_make1(makeDefElem("oids", (Node *) makeInteger(false), @1)); } | /*EMPTY*/ { $$ = NIL; } ; @@ -3636,51 +3636,51 @@ SeqOptList: SeqOptElem { $$ = list_make1($1); } SeqOptElem: CACHE NumericOnly { - $$ = makeDefElem("cache", (Node *)$2); + $$ = makeDefElem("cache", (Node *)$2, @1); } | CYCLE { - $$ = makeDefElem("cycle", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("cycle", (Node *)makeInteger(TRUE), @1); } | NO CYCLE { - $$ = makeDefElem("cycle", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("cycle", (Node *)makeInteger(FALSE), @1); } | INCREMENT opt_by NumericOnly { - $$ = makeDefElem("increment", (Node *)$3); + $$ = makeDefElem("increment", (Node *)$3, @1); } | MAXVALUE NumericOnly { - $$ = makeDefElem("maxvalue", (Node *)$2); + $$ = makeDefElem("maxvalue", (Node *)$2, @1); } | MINVALUE NumericOnly { - $$ = makeDefElem("minvalue", (Node *)$2); + $$ = makeDefElem("minvalue", (Node *)$2, @1); } | NO MAXVALUE { - $$ = makeDefElem("maxvalue", NULL); + $$ = makeDefElem("maxvalue", NULL, @1); } | NO MINVALUE { - $$ = makeDefElem("minvalue", NULL); + $$ = makeDefElem("minvalue", NULL, @1); } | OWNED BY any_name { - $$ = makeDefElem("owned_by", (Node *)$3); + $$ = makeDefElem("owned_by", (Node *)$3, @1); } | START opt_with NumericOnly { - $$ = makeDefElem("start", (Node *)$3); + $$ = makeDefElem("start", (Node *)$3, @1); } | RESTART { - $$ = makeDefElem("restart", NULL); + $$ = makeDefElem("restart", NULL, @1); } | RESTART opt_with NumericOnly { - $$ = makeDefElem("restart", (Node *)$3); + $$ = makeDefElem("restart", (Node *)$3, @1); } ; @@ -3879,19 +3879,19 @@ create_extension_opt_list: create_extension_opt_item: SCHEMA name { - $$ = makeDefElem("schema", (Node *)makeString($2)); + $$ = makeDefElem("schema", (Node *)makeString($2), @1); } | VERSION_P NonReservedWord_or_Sconst { - $$ = makeDefElem("new_version", (Node *)makeString($2)); + $$ = makeDefElem("new_version", (Node *)makeString($2), @1); } | FROM NonReservedWord_or_Sconst { - $$ = makeDefElem("old_version", (Node *)makeString($2)); + $$ = makeDefElem("old_version", (Node *)makeString($2), @1); } | CASCADE { - $$ = makeDefElem("cascade", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("cascade", (Node *)makeInteger(TRUE), @1); } ; @@ -3920,7 +3920,7 @@ alter_extension_opt_list: alter_extension_opt_item: TO NonReservedWord_or_Sconst { - $$ = makeDefElem("new_version", (Node *)makeString($2)); + $$ = makeDefElem("new_version", (Node *)makeString($2), @1); } ; @@ -4181,10 +4181,10 @@ CreateFdwStmt: CREATE FOREIGN DATA_P WRAPPER name opt_fdw_options create_generic ; fdw_option: - HANDLER handler_name { $$ = makeDefElem("handler", (Node *)$2); } - | NO HANDLER { $$ = makeDefElem("handler", NULL); } - | VALIDATOR handler_name { $$ = makeDefElem("validator", (Node *)$2); } - | NO VALIDATOR { $$ = makeDefElem("validator", NULL); } + HANDLER handler_name { $$ = makeDefElem("handler", (Node *)$2, @1); } + | NO HANDLER { $$ = makeDefElem("handler", NULL, @1); } + | VALIDATOR handler_name { $$ = makeDefElem("validator", (Node *)$2, @1); } + | NO VALIDATOR { $$ = makeDefElem("validator", NULL, @1); } ; fdw_options: @@ -4303,14 +4303,14 @@ alter_generic_option_elem: } | DROP generic_option_name { - $$ = makeDefElemExtended(NULL, $2, NULL, DEFELEM_DROP); + $$ = makeDefElemExtended(NULL, $2, NULL, DEFELEM_DROP, @2); } ; generic_option_elem: generic_option_name generic_option_arg { - $$ = makeDefElem($1, $2); + $$ = makeDefElem($1, $2, @1); } ; @@ -4982,7 +4982,7 @@ event_trigger_when_list: event_trigger_when_item: ColId IN_P '(' event_trigger_value_list ')' - { $$ = makeDefElem($1, (Node *) $4); } + { $$ = makeDefElem($1, (Node *) $4, @1); } ; event_trigger_value_list: @@ -5187,7 +5187,7 @@ DefineStmt: n->kind = OBJECT_COLLATION; n->args = NIL; n->defnames = $3; - n->definition = list_make1(makeDefElem("from", (Node *) $5)); + n->definition = list_make1(makeDefElem("from", (Node *) $5, @5)); $$ = (Node *)n; } ; @@ -5201,11 +5201,11 @@ def_list: def_elem { $$ = list_make1($1); } def_elem: ColLabel '=' def_arg { - $$ = makeDefElem($1, (Node *) $3); + $$ = makeDefElem($1, (Node *) $3, @1); } | ColLabel { - $$ = makeDefElem($1, NULL); + $$ = makeDefElem($1, NULL, @1); } ; @@ -5231,7 +5231,7 @@ old_aggr_list: old_aggr_elem { $$ = list_make1($1); } */ old_aggr_elem: IDENT '=' def_arg { - $$ = makeDefElem($1, (Node *)$3); + $$ = makeDefElem($1, (Node *)$3, @1); } ; @@ -6552,15 +6552,15 @@ DefACLOptionList: DefACLOption: IN_P SCHEMA name_list { - $$ = makeDefElem("schemas", (Node *)$3); + $$ = makeDefElem("schemas", (Node *)$3, @1); } | FOR ROLE role_list { - $$ = makeDefElem("roles", (Node *)$3); + $$ = makeDefElem("roles", (Node *)$3, @1); } | FOR USER role_list { - $$ = makeDefElem("roles", (Node *)$3); + $$ = makeDefElem("roles", (Node *)$3, @1); } ; @@ -7044,87 +7044,87 @@ createfunc_opt_list: common_func_opt_item: CALLED ON NULL_P INPUT_P { - $$ = makeDefElem("strict", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("strict", (Node *)makeInteger(FALSE), @1); } | RETURNS NULL_P ON NULL_P INPUT_P { - $$ = makeDefElem("strict", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("strict", (Node *)makeInteger(TRUE), @1); } | STRICT_P { - $$ = makeDefElem("strict", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("strict", (Node *)makeInteger(TRUE), @1); } | IMMUTABLE { - $$ = makeDefElem("volatility", (Node *)makeString("immutable")); + $$ = makeDefElem("volatility", (Node *)makeString("immutable"), @1); } | STABLE { - $$ = makeDefElem("volatility", (Node *)makeString("stable")); + $$ = makeDefElem("volatility", (Node *)makeString("stable"), @1); } | VOLATILE { - $$ = makeDefElem("volatility", (Node *)makeString("volatile")); + $$ = makeDefElem("volatility", (Node *)makeString("volatile"), @1); } | EXTERNAL SECURITY DEFINER { - $$ = makeDefElem("security", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("security", (Node *)makeInteger(TRUE), @1); } | EXTERNAL SECURITY INVOKER { - $$ = makeDefElem("security", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("security", (Node *)makeInteger(FALSE), @1); } | SECURITY DEFINER { - $$ = makeDefElem("security", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("security", (Node *)makeInteger(TRUE), @1); } | SECURITY INVOKER { - $$ = makeDefElem("security", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("security", (Node *)makeInteger(FALSE), @1); } | LEAKPROOF { - $$ = makeDefElem("leakproof", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("leakproof", (Node *)makeInteger(TRUE), @1); } | NOT LEAKPROOF { - $$ = makeDefElem("leakproof", (Node *)makeInteger(FALSE)); + $$ = makeDefElem("leakproof", (Node *)makeInteger(FALSE), @1); } | COST NumericOnly { - $$ = makeDefElem("cost", (Node *)$2); + $$ = makeDefElem("cost", (Node *)$2, @1); } | ROWS NumericOnly { - $$ = makeDefElem("rows", (Node *)$2); + $$ = makeDefElem("rows", (Node *)$2, @1); } | FunctionSetResetClause { /* we abuse the normal content of a DefElem here */ - $$ = makeDefElem("set", (Node *)$1); + $$ = makeDefElem("set", (Node *)$1, @1); } | PARALLEL ColId { - $$ = makeDefElem("parallel", (Node *)makeString($2)); + $$ = makeDefElem("parallel", (Node *)makeString($2), @1); } ; createfunc_opt_item: AS func_as { - $$ = makeDefElem("as", (Node *)$2); + $$ = makeDefElem("as", (Node *)$2, @1); } | LANGUAGE NonReservedWord_or_Sconst { - $$ = makeDefElem("language", (Node *)makeString($2)); + $$ = makeDefElem("language", (Node *)makeString($2), @1); } | TRANSFORM transform_type_list { - $$ = makeDefElem("transform", (Node *)$2); + $$ = makeDefElem("transform", (Node *)$2, @1); } | WINDOW { - $$ = makeDefElem("window", (Node *)makeInteger(TRUE)); + $$ = makeDefElem("window", (Node *)makeInteger(TRUE), @1); } | common_func_opt_item { @@ -7336,11 +7336,11 @@ dostmt_opt_list: dostmt_opt_item: Sconst { - $$ = makeDefElem("as", (Node *)makeString($1)); + $$ = makeDefElem("as", (Node *)makeString($1), @1); } | LANGUAGE NonReservedWord_or_Sconst { - $$ = makeDefElem("language", (Node *)makeString($2)); + $$ = makeDefElem("language", (Node *)makeString($2), @1); } ; @@ -8329,9 +8329,9 @@ operator_def_list: operator_def_elem { $$ = list_make1($1); } ; operator_def_elem: ColLabel '=' NONE - { $$ = makeDefElem($1, NULL); } + { $$ = makeDefElem($1, NULL, @1); } | ColLabel '=' def_arg - { $$ = makeDefElem($1, (Node *) $3); } + { $$ = makeDefElem($1, (Node *) $3, @1); } ; /***************************************************************************** @@ -8697,7 +8697,7 @@ TransactionStmt: TransactionStmt *n = makeNode(TransactionStmt); n->kind = TRANS_STMT_SAVEPOINT; n->options = list_make1(makeDefElem("savepoint_name", - (Node *)makeString($2))); + (Node *)makeString($2), @1)); $$ = (Node *)n; } | RELEASE SAVEPOINT ColId @@ -8705,7 +8705,7 @@ TransactionStmt: TransactionStmt *n = makeNode(TransactionStmt); n->kind = TRANS_STMT_RELEASE; n->options = list_make1(makeDefElem("savepoint_name", - (Node *)makeString($3))); + (Node *)makeString($3), @1)); $$ = (Node *)n; } | RELEASE ColId @@ -8713,7 +8713,7 @@ TransactionStmt: TransactionStmt *n = makeNode(TransactionStmt); n->kind = TRANS_STMT_RELEASE; n->options = list_make1(makeDefElem("savepoint_name", - (Node *)makeString($2))); + (Node *)makeString($2), @1)); $$ = (Node *)n; } | ROLLBACK opt_transaction TO SAVEPOINT ColId @@ -8721,7 +8721,7 @@ TransactionStmt: TransactionStmt *n = makeNode(TransactionStmt); n->kind = TRANS_STMT_ROLLBACK_TO; n->options = list_make1(makeDefElem("savepoint_name", - (Node *)makeString($5))); + (Node *)makeString($5), @1)); $$ = (Node *)n; } | ROLLBACK opt_transaction TO ColId @@ -8729,7 +8729,7 @@ TransactionStmt: TransactionStmt *n = makeNode(TransactionStmt); n->kind = TRANS_STMT_ROLLBACK_TO; n->options = list_make1(makeDefElem("savepoint_name", - (Node *)makeString($4))); + (Node *)makeString($4), @1)); $$ = (Node *)n; } | PREPARE TRANSACTION Sconst @@ -8763,19 +8763,19 @@ opt_transaction: WORK {} transaction_mode_item: ISOLATION LEVEL iso_level { $$ = makeDefElem("transaction_isolation", - makeStringConst($3, @3)); } + makeStringConst($3, @3), @1); } | READ ONLY { $$ = makeDefElem("transaction_read_only", - makeIntConst(TRUE, @1)); } + makeIntConst(TRUE, @1), @1); } | READ WRITE { $$ = makeDefElem("transaction_read_only", - makeIntConst(FALSE, @1)); } + makeIntConst(FALSE, @1), @1); } | DEFERRABLE { $$ = makeDefElem("transaction_deferrable", - makeIntConst(TRUE, @1)); } + makeIntConst(TRUE, @1), @1); } | NOT DEFERRABLE { $$ = makeDefElem("transaction_deferrable", - makeIntConst(FALSE, @1)); } + makeIntConst(FALSE, @1), @1); } ; /* Syntax with commas is SQL-spec, without commas is Postgres historical */ @@ -8919,15 +8919,15 @@ createdb_opt_items: createdb_opt_item: createdb_opt_name opt_equal SignedIconst { - $$ = makeDefElem($1, (Node *)makeInteger($3)); + $$ = makeDefElem($1, (Node *)makeInteger($3), @1); } | createdb_opt_name opt_equal opt_boolean_or_string { - $$ = makeDefElem($1, (Node *)makeString($3)); + $$ = makeDefElem($1, (Node *)makeString($3), @1); } | createdb_opt_name opt_equal DEFAULT { - $$ = makeDefElem($1, NULL); + $$ = makeDefElem($1, NULL, @1); } ; @@ -8987,7 +8987,7 @@ AlterDatabaseStmt: AlterDatabaseStmt *n = makeNode(AlterDatabaseStmt); n->dbname = $3; n->options = list_make1(makeDefElem("tablespace", - (Node *)makeString($6))); + (Node *)makeString($6), @6)); $$ = (Node *)n; } ; @@ -9451,17 +9451,17 @@ ExplainStmt: { ExplainStmt *n = makeNode(ExplainStmt); n->query = $4; - n->options = list_make1(makeDefElem("analyze", NULL)); + n->options = list_make1(makeDefElem("analyze", NULL, @2)); if ($3) n->options = lappend(n->options, - makeDefElem("verbose", NULL)); + makeDefElem("verbose", NULL, @3)); $$ = (Node *) n; } | EXPLAIN VERBOSE ExplainableStmt { ExplainStmt *n = makeNode(ExplainStmt); n->query = $3; - n->options = list_make1(makeDefElem("verbose", NULL)); + n->options = list_make1(makeDefElem("verbose", NULL, @2)); $$ = (Node *) n; } | EXPLAIN '(' explain_option_list ')' ExplainableStmt @@ -9499,7 +9499,7 @@ explain_option_list: explain_option_elem: explain_option_name explain_option_arg { - $$ = makeDefElem($1, $2); + $$ = makeDefElem($1, $2, @1); } ; diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index e98fad051e..7a2950e6a9 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -294,7 +294,8 @@ transformCreateStmt(CreateStmt *stmt, const char *queryString) * overridden if an inherited table has oids. */ stmt->options = lcons(makeDefElem("oids", - (Node *) makeInteger(cxt.hasoids)), stmt->options); + (Node *) makeInteger(cxt.hasoids), -1), + stmt->options); } foreach(elements, stmt->tableElts) @@ -482,7 +483,7 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column) makeString(cxt->relation->relname), makeString(column->colname)); altseqstmt->options = list_make1(makeDefElem("owned_by", - (Node *) attnamelist)); + (Node *) attnamelist, -1)); cxt->alist = lappend(cxt->alist, altseqstmt); diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 9c7be2dc7b..318726ebf3 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -225,7 +225,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin char *name = TextDatumGetCString(datum_opts[i]); char *opt = TextDatumGetCString(datum_opts[i + 1]); - options = lappend(options, makeDefElem(name, (Node *) makeString(opt))); + options = lappend(options, makeDefElem(name, (Node *) makeString(opt), -1)); } } diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index d93db88d42..fd0fa6dde0 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -148,37 +148,37 @@ base_backup_opt: K_LABEL SCONST { $$ = makeDefElem("label", - (Node *)makeString($2)); + (Node *)makeString($2), -1); } | K_PROGRESS { $$ = makeDefElem("progress", - (Node *)makeInteger(TRUE)); + (Node *)makeInteger(TRUE), -1); } | K_FAST { $$ = makeDefElem("fast", - (Node *)makeInteger(TRUE)); + (Node *)makeInteger(TRUE), -1); } | K_WAL { $$ = makeDefElem("wal", - (Node *)makeInteger(TRUE)); + (Node *)makeInteger(TRUE), -1); } | K_NOWAIT { $$ = makeDefElem("nowait", - (Node *)makeInteger(TRUE)); + (Node *)makeInteger(TRUE), -1); } | K_MAX_RATE UCONST { $$ = makeDefElem("max_rate", - (Node *)makeInteger($2)); + (Node *)makeInteger($2), -1); } | K_TABLESPACE_MAP { $$ = makeDefElem("tablespace_map", - (Node *)makeInteger(TRUE)); + (Node *)makeInteger(TRUE), -1); } ; @@ -315,7 +315,7 @@ plugin_opt_list: plugin_opt_elem: IDENT plugin_opt_arg { - $$ = makeDefElem($1, $2); + $$ = makeDefElem($1, $2, -1); } ; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index ac64135d5d..f50ce408ae 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -71,7 +71,8 @@ ProcessUtility_hook_type ProcessUtility_hook = NULL; /* local function declarations */ -static void ProcessUtilitySlow(Node *parsetree, +static void ProcessUtilitySlow(ParseState *pstate, + Node *parsetree, const char *queryString, ProcessUtilityContext context, ParamListInfo params, @@ -358,12 +359,16 @@ standard_ProcessUtility(Node *parsetree, char *completionTag) { bool isTopLevel = (context == PROCESS_UTILITY_TOPLEVEL); + ParseState *pstate; check_xact_readonly(parsetree); if (completionTag) completionTag[0] = '\0'; + pstate = make_parsestate(NULL); + pstate->p_sourcetext = queryString; + switch (nodeTag(parsetree)) { /* @@ -540,7 +545,7 @@ standard_ProcessUtility(Node *parsetree, { uint64 processed; - DoCopy((CopyStmt *) parsetree, queryString, &processed); + DoCopy(pstate, (CopyStmt *) parsetree, &processed); if (completionTag) snprintf(completionTag, COMPLETION_TAG_BUFSIZE, "COPY " UINT64_FORMAT, processed); @@ -571,12 +576,12 @@ standard_ProcessUtility(Node *parsetree, case T_CreatedbStmt: /* no event triggers for global objects */ PreventTransactionChain(isTopLevel, "CREATE DATABASE"); - createdb((CreatedbStmt *) parsetree); + createdb(pstate, (CreatedbStmt *) parsetree); break; case T_AlterDatabaseStmt: /* no event triggers for global objects */ - AlterDatabase((AlterDatabaseStmt *) parsetree, isTopLevel); + AlterDatabase(pstate, (AlterDatabaseStmt *) parsetree, isTopLevel); break; case T_AlterDatabaseSetStmt: @@ -657,7 +662,7 @@ standard_ProcessUtility(Node *parsetree, break; case T_ExplainStmt: - ExplainQuery((ExplainStmt *) parsetree, queryString, params, dest); + ExplainQuery(pstate, (ExplainStmt *) parsetree, queryString, params, dest); break; case T_AlterSystemStmt: @@ -698,7 +703,7 @@ standard_ProcessUtility(Node *parsetree, */ case T_CreateRoleStmt: /* no event triggers for global objects */ - CreateRole((CreateRoleStmt *) parsetree); + CreateRole(pstate, (CreateRoleStmt *) parsetree); break; case T_AlterRoleStmt: @@ -803,7 +808,7 @@ standard_ProcessUtility(Node *parsetree, GrantStmt *stmt = (GrantStmt *) parsetree; if (EventTriggerSupportsGrantObjectType(stmt->objtype)) - ProcessUtilitySlow(parsetree, queryString, + ProcessUtilitySlow(pstate, parsetree, queryString, context, params, dest, completionTag); else @@ -816,7 +821,7 @@ standard_ProcessUtility(Node *parsetree, DropStmt *stmt = (DropStmt *) parsetree; if (EventTriggerSupportsObjectType(stmt->removeType)) - ProcessUtilitySlow(parsetree, queryString, + ProcessUtilitySlow(pstate, parsetree, queryString, context, params, dest, completionTag); else @@ -829,7 +834,7 @@ standard_ProcessUtility(Node *parsetree, RenameStmt *stmt = (RenameStmt *) parsetree; if (EventTriggerSupportsObjectType(stmt->renameType)) - ProcessUtilitySlow(parsetree, queryString, + ProcessUtilitySlow(pstate, parsetree, queryString, context, params, dest, completionTag); else @@ -842,7 +847,7 @@ standard_ProcessUtility(Node *parsetree, AlterObjectDependsStmt *stmt = (AlterObjectDependsStmt *) parsetree; if (EventTriggerSupportsObjectType(stmt->objectType)) - ProcessUtilitySlow(parsetree, queryString, + ProcessUtilitySlow(pstate, parsetree, queryString, context, params, dest, completionTag); else @@ -855,7 +860,7 @@ standard_ProcessUtility(Node *parsetree, AlterObjectSchemaStmt *stmt = (AlterObjectSchemaStmt *) parsetree; if (EventTriggerSupportsObjectType(stmt->objectType)) - ProcessUtilitySlow(parsetree, queryString, + ProcessUtilitySlow(pstate, parsetree, queryString, context, params, dest, completionTag); else @@ -868,7 +873,7 @@ standard_ProcessUtility(Node *parsetree, AlterOwnerStmt *stmt = (AlterOwnerStmt *) parsetree; if (EventTriggerSupportsObjectType(stmt->objectType)) - ProcessUtilitySlow(parsetree, queryString, + ProcessUtilitySlow(pstate, parsetree, queryString, context, params, dest, completionTag); else @@ -881,7 +886,7 @@ standard_ProcessUtility(Node *parsetree, CommentStmt *stmt = (CommentStmt *) parsetree; if (EventTriggerSupportsObjectType(stmt->objtype)) - ProcessUtilitySlow(parsetree, queryString, + ProcessUtilitySlow(pstate, parsetree, queryString, context, params, dest, completionTag); else @@ -894,7 +899,7 @@ standard_ProcessUtility(Node *parsetree, SecLabelStmt *stmt = (SecLabelStmt *) parsetree; if (EventTriggerSupportsObjectType(stmt->objtype)) - ProcessUtilitySlow(parsetree, queryString, + ProcessUtilitySlow(pstate, parsetree, queryString, context, params, dest, completionTag); else @@ -904,11 +909,13 @@ standard_ProcessUtility(Node *parsetree, default: /* All other statement types have event trigger support */ - ProcessUtilitySlow(parsetree, queryString, + ProcessUtilitySlow(pstate, parsetree, queryString, context, params, dest, completionTag); break; } + + free_parsestate(pstate); } /* @@ -917,7 +924,8 @@ standard_ProcessUtility(Node *parsetree, * perform the trigger support calls if the context allows it. */ static void -ProcessUtilitySlow(Node *parsetree, +ProcessUtilitySlow(ParseState *pstate, + Node *parsetree, const char *queryString, ProcessUtilityContext context, ParamListInfo params, @@ -1191,9 +1199,9 @@ ProcessUtilitySlow(Node *parsetree, { case OBJECT_AGGREGATE: address = - DefineAggregate(stmt->defnames, stmt->args, + DefineAggregate(pstate, stmt->defnames, stmt->args, stmt->oldstyle, - stmt->definition, queryString); + stmt->definition); break; case OBJECT_OPERATOR: Assert(stmt->args == NIL); @@ -1202,7 +1210,8 @@ ProcessUtilitySlow(Node *parsetree, break; case OBJECT_TYPE: Assert(stmt->args == NIL); - address = DefineType(stmt->defnames, + address = DefineType(pstate, + stmt->defnames, stmt->definition); break; case OBJECT_TSPARSER: @@ -1228,7 +1237,8 @@ ProcessUtilitySlow(Node *parsetree, break; case OBJECT_COLLATION: Assert(stmt->args == NIL); - address = DefineCollation(stmt->defnames, + address = DefineCollation(pstate, + stmt->defnames, stmt->definition); break; default: @@ -1293,11 +1303,11 @@ ProcessUtilitySlow(Node *parsetree, break; case T_CreateExtensionStmt: - address = CreateExtension((CreateExtensionStmt *) parsetree); + address = CreateExtension(pstate, (CreateExtensionStmt *) parsetree); break; case T_AlterExtensionStmt: - address = ExecAlterExtensionStmt((AlterExtensionStmt *) parsetree); + address = ExecAlterExtensionStmt(pstate, (AlterExtensionStmt *) parsetree); break; case T_AlterExtensionContentsStmt: @@ -1373,11 +1383,11 @@ ProcessUtilitySlow(Node *parsetree, break; case T_CreateFunctionStmt: /* CREATE FUNCTION */ - address = CreateFunction((CreateFunctionStmt *) parsetree, queryString); + address = CreateFunction(pstate, (CreateFunctionStmt *) parsetree); break; case T_AlterFunctionStmt: /* ALTER FUNCTION */ - address = AlterFunction((AlterFunctionStmt *) parsetree); + address = AlterFunction(pstate, (AlterFunctionStmt *) parsetree); break; case T_RuleStmt: /* CREATE RULE */ @@ -1385,11 +1395,11 @@ ProcessUtilitySlow(Node *parsetree, break; case T_CreateSeqStmt: - address = DefineSequence((CreateSeqStmt *) parsetree); + address = DefineSequence(pstate, (CreateSeqStmt *) parsetree); break; case T_AlterSeqStmt: - address = AlterSequence((AlterSeqStmt *) parsetree); + address = AlterSequence(pstate, (AlterSeqStmt *) parsetree); break; case T_CreateTableAsStmt: @@ -1523,7 +1533,7 @@ ProcessUtilitySlow(Node *parsetree, break; case T_AlterDefaultPrivilegesStmt: - ExecAlterDefaultPrivilegesStmt((AlterDefaultPrivilegesStmt *) parsetree); + ExecAlterDefaultPrivilegesStmt(pstate, (AlterDefaultPrivilegesStmt *) parsetree); EventTriggerCollectAlterDefPrivs((AlterDefaultPrivilegesStmt *) parsetree); commandCollected = true; break; diff --git a/src/include/commands/collationcmds.h b/src/include/commands/collationcmds.h index d1e5e0ad84..073314e76d 100644 --- a/src/include/commands/collationcmds.h +++ b/src/include/commands/collationcmds.h @@ -18,7 +18,7 @@ #include "catalog/objectaddress.h" #include "nodes/parsenodes.h" -extern ObjectAddress DefineCollation(List *names, List *parameters); +extern ObjectAddress DefineCollation(ParseState *pstate, List *names, List *parameters); extern void IsThereCollationInNamespace(const char *collname, Oid nspOid); #endif /* COLLATIONCMDS_H */ diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 314d1f7e27..65eb347b3f 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -16,16 +16,17 @@ #include "nodes/execnodes.h" #include "nodes/parsenodes.h" +#include "parser/parse_node.h" #include "tcop/dest.h" /* CopyStateData is private in commands/copy.c */ typedef struct CopyStateData *CopyState; -extern Oid DoCopy(const CopyStmt *stmt, const char *queryString, +extern Oid DoCopy(ParseState *state, const CopyStmt *stmt, uint64 *processed); -extern void ProcessCopyOptions(CopyState cstate, bool is_from, List *options); -extern CopyState BeginCopyFrom(Relation rel, const char *filename, +extern void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options); +extern CopyState BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename, bool is_program, List *attnamelist, List *options); extern void EndCopyFrom(CopyState cstate); extern bool NextCopyFrom(CopyState cstate, ExprContext *econtext, diff --git a/src/include/commands/dbcommands.h b/src/include/commands/dbcommands.h index b6436f1812..9bea085e1f 100644 --- a/src/include/commands/dbcommands.h +++ b/src/include/commands/dbcommands.h @@ -19,10 +19,10 @@ #include "lib/stringinfo.h" #include "nodes/parsenodes.h" -extern Oid createdb(const CreatedbStmt *stmt); +extern Oid createdb(ParseState *pstate, const CreatedbStmt *stmt); extern void dropdb(const char *dbname, bool missing_ok); extern ObjectAddress RenameDatabase(const char *oldname, const char *newname); -extern Oid AlterDatabase(AlterDatabaseStmt *stmt, bool isTopLevel); +extern Oid AlterDatabase(ParseState *pstate, AlterDatabaseStmt *stmt, bool isTopLevel); extern Oid AlterDatabaseSet(AlterDatabaseSetStmt *stmt); extern ObjectAddress AlterDatabaseOwner(const char *dbname, Oid newOwnerId); diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index b064eb4836..2b894ff262 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -44,11 +44,11 @@ extern bool CheckIndexCompatible(Oid oldId, extern Oid GetDefaultOpClass(Oid type_id, Oid am_id); /* commands/functioncmds.c */ -extern ObjectAddress CreateFunction(CreateFunctionStmt *stmt, const char *queryString); +extern ObjectAddress CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt); extern void RemoveFunctionById(Oid funcOid); extern void SetFunctionReturnType(Oid funcOid, Oid newRetType); extern void SetFunctionArgType(Oid funcOid, int argIndex, Oid newArgType); -extern ObjectAddress AlterFunction(AlterFunctionStmt *stmt); +extern ObjectAddress AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt); extern ObjectAddress CreateCast(CreateCastStmt *stmt); extern void DropCastById(Oid castOid); extern ObjectAddress CreateTransform(CreateTransformStmt *stmt); @@ -58,10 +58,10 @@ extern void IsThereFunctionInNamespace(const char *proname, int pronargs, extern void ExecuteDoStmt(DoStmt *stmt); extern Oid get_cast_oid(Oid sourcetypeid, Oid targettypeid, bool missing_ok); extern Oid get_transform_oid(Oid type_id, Oid lang_id, bool missing_ok); -extern void interpret_function_parameter_list(List *parameters, +extern void interpret_function_parameter_list(ParseState *pstate, + List *parameters, Oid languageOid, bool is_aggregate, - const char *queryString, oidvector **parameterTypes, ArrayType **allParameterTypes, ArrayType **parameterModes, @@ -76,8 +76,8 @@ extern void RemoveOperatorById(Oid operOid); extern ObjectAddress AlterOperator(AlterOperatorStmt *stmt); /* commands/aggregatecmds.c */ -extern ObjectAddress DefineAggregate(List *name, List *args, bool oldstyle, - List *parameters, const char *queryString); +extern ObjectAddress DefineAggregate(ParseState *pstate, List *name, List *args, bool oldstyle, + List *parameters); /* commands/opclasscmds.c */ extern ObjectAddress DefineOpClass(CreateOpClassStmt *stmt); @@ -152,6 +152,5 @@ extern int64 defGetInt64(DefElem *def); extern List *defGetQualifiedName(DefElem *def); extern TypeName *defGetTypeName(DefElem *def); extern int defGetTypeLength(DefElem *def); -extern DefElem *defWithOids(bool value); #endif /* DEFREM_H */ diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index 3d0a5abbc2..8b3acab0f2 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -15,6 +15,7 @@ #include "executor/executor.h" #include "lib/stringinfo.h" +#include "parser/parse_node.h" typedef enum ExplainFormat { @@ -59,7 +60,7 @@ typedef const char *(*explain_get_index_name_hook_type) (Oid indexId); extern PGDLLIMPORT explain_get_index_name_hook_type explain_get_index_name_hook; -extern void ExplainQuery(ExplainStmt *stmt, const char *queryString, +extern void ExplainQuery(ParseState *pstate, ExplainStmt *stmt, const char *queryString, ParamListInfo params, DestReceiver *dest); extern ExplainState *NewExplainState(void); diff --git a/src/include/commands/extension.h b/src/include/commands/extension.h index 94354ab04d..e98b245ac1 100644 --- a/src/include/commands/extension.h +++ b/src/include/commands/extension.h @@ -28,7 +28,7 @@ extern PGDLLIMPORT bool creating_extension; extern Oid CurrentExtensionObject; -extern ObjectAddress CreateExtension(CreateExtensionStmt *stmt); +extern ObjectAddress CreateExtension(ParseState *pstate, CreateExtensionStmt *stmt); extern void RemoveExtensionById(Oid extId); @@ -37,7 +37,7 @@ extern ObjectAddress InsertExtensionTuple(const char *extName, Oid extOwner, Datum extConfig, Datum extCondition, List *requiredExtensions); -extern ObjectAddress ExecAlterExtensionStmt(AlterExtensionStmt *stmt); +extern ObjectAddress ExecAlterExtensionStmt(ParseState *pstate, AlterExtensionStmt *stmt); extern ObjectAddress ExecAlterExtensionContentsStmt(AlterExtensionContentsStmt *stmt, ObjectAddress *objAddress); diff --git a/src/include/commands/sequence.h b/src/include/commands/sequence.h index 6af60d893b..392a626508 100644 --- a/src/include/commands/sequence.h +++ b/src/include/commands/sequence.h @@ -18,6 +18,7 @@ #include "fmgr.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" +#include "parser/parse_node.h" #include "storage/relfilenode.h" @@ -73,8 +74,8 @@ extern Datum lastval(PG_FUNCTION_ARGS); extern Datum pg_sequence_parameters(PG_FUNCTION_ARGS); -extern ObjectAddress DefineSequence(CreateSeqStmt *stmt); -extern ObjectAddress AlterSequence(AlterSeqStmt *stmt); +extern ObjectAddress DefineSequence(ParseState *pstate, CreateSeqStmt *stmt); +extern ObjectAddress AlterSequence(ParseState *pstate, AlterSeqStmt *stmt); extern void ResetSequence(Oid seq_relid); extern void ResetSequenceCaches(void); diff --git a/src/include/commands/typecmds.h b/src/include/commands/typecmds.h index 847b770f00..56b4cecd28 100644 --- a/src/include/commands/typecmds.h +++ b/src/include/commands/typecmds.h @@ -21,7 +21,7 @@ #define DEFAULT_TYPDELIM ',' -extern ObjectAddress DefineType(List *names, List *parameters); +extern ObjectAddress DefineType(ParseState *pstate, List *names, List *parameters); extern void RemoveTypeById(Oid typeOid); extern ObjectAddress DefineDomain(CreateDomainStmt *stmt); extern ObjectAddress DefineEnum(CreateEnumStmt *stmt); diff --git a/src/include/commands/user.h b/src/include/commands/user.h index d35cb0c90d..1f0cfcc86f 100644 --- a/src/include/commands/user.h +++ b/src/include/commands/user.h @@ -13,6 +13,7 @@ #include "catalog/objectaddress.h" #include "nodes/parsenodes.h" +#include "parser/parse_node.h" /* Hook to check passwords in CreateRole() and AlterRole() */ @@ -23,7 +24,7 @@ typedef void (*check_password_hook_type) (const char *username, const char *pass extern PGDLLIMPORT check_password_hook_type check_password_hook; -extern Oid CreateRole(CreateRoleStmt *stmt); +extern Oid CreateRole(ParseState *pstate, CreateRoleStmt *stmt); extern Oid AlterRole(AlterRoleStmt *stmt); extern Oid AlterRoleSet(AlterRoleSetStmt *stmt); extern void DropRole(DropRoleStmt *stmt); diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h index 01c5cb45d5..47500cb44b 100644 --- a/src/include/nodes/makefuncs.h +++ b/src/include/nodes/makefuncs.h @@ -80,9 +80,9 @@ extern FuncExpr *makeFuncExpr(Oid funcid, Oid rettype, List *args, extern FuncCall *makeFuncCall(List *name, List *args, int location); -extern DefElem *makeDefElem(char *name, Node *arg); +extern DefElem *makeDefElem(char *name, Node *arg, int location); extern DefElem *makeDefElemExtended(char *nameSpace, char *name, Node *arg, - DefElemAction defaction); + DefElemAction defaction, int location); extern GroupingSet *makeGroupingSet(GroupingSetKind kind, List *content, int location); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 1481fff57d..3716c2eef9 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -666,6 +666,7 @@ typedef struct DefElem char *defname; Node *arg; /* a (Value *) or a (TypeName *) */ DefElemAction defaction; /* unspecified action, or SET/ADD/DROP */ + int location; /* token location, or -1 if unknown */ } DefElem; /* diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 4cc49f0c0c..fda75bb618 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -26,6 +26,7 @@ #include "access/htup.h" #include "nodes/parsenodes.h" +#include "parser/parse_node.h" #include "utils/array.h" #include "utils/snapshot.h" @@ -259,7 +260,7 @@ extern Datum aclexplode(PG_FUNCTION_ARGS); * prototypes for functions in aclchk.c */ extern void ExecuteGrantStmt(GrantStmt *stmt); -extern void ExecAlterDefaultPrivilegesStmt(AlterDefaultPrivilegesStmt *stmt); +extern void ExecAlterDefaultPrivilegesStmt(ParseState *pstate, AlterDefaultPrivilegesStmt *stmt); extern void RemoveRoleFromObjectACL(Oid roleid, Oid classid, Oid objid); extern void RemoveDefaultACLById(Oid defaclOid); From 4f405c8ef4704b7fa7fd8ac14a66c5f5d13722c4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 6 Sep 2016 23:55:55 -0400 Subject: [PATCH 128/871] Add a HINT for client-vs-server COPY failure cases. Users often get confused between COPY and \copy and try to use client-side paths with COPY. The server then cannot find the file (if remote), or sees a permissions problem (if local), or some variant of that. Emit a hint about this in the most common cases. In future we might want to expand the set of errnos for which the hint gets printed, but be conservative for now. Craig Ringer, reviewed by Christoph Berg and Tom Lane Discussion: --- src/backend/commands/copy.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index e393c0b39e..432b0ca67b 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -280,7 +280,7 @@ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"; /* non-export function prototypes */ static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel, Node *raw_query, - const Oid queryRelId, List *attnamelist, + const Oid queryRelId, List *attnamelist, List *options); static void EndCopy(CopyState cstate); static void ClosePipeToProgram(CopyState cstate); @@ -1175,7 +1175,7 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("option \"%s\" not recognized", defel->defname), - parser_errposition(pstate, defel->location))); + parser_errposition(pstate, defel->location))); } /* @@ -1785,10 +1785,18 @@ BeginCopyTo(ParseState *pstate, cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W); umask(oumask); if (cstate->copy_file == NULL) + { + /* copy errno because ereport subfunctions might change it */ + int save_errno = errno; + ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\" for writing: %m", - cstate->filename))); + cstate->filename), + (save_errno == ENOENT || save_errno == EACCES) ? + errhint("COPY TO instructs the PostgreSQL server process to write a file. " + "You may want a client-side facility such as psql's \\copy.") : 0)); + } if (fstat(fileno(cstate->copy_file), &st)) ereport(ERROR, @@ -2810,10 +2818,18 @@ BeginCopyFrom(ParseState *pstate, cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R); if (cstate->copy_file == NULL) + { + /* copy errno because ereport subfunctions might change it */ + int save_errno = errno; + ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\" for reading: %m", - cstate->filename))); + cstate->filename), + (save_errno == ENOENT || save_errno == EACCES) ? + errhint("COPY FROM instructs the PostgreSQL server process to read a file. " + "You may want a client-side facility such as psql's \\copy.") : 0)); + } if (fstat(fileno(cstate->copy_file), &st)) ereport(ERROR, From bd180b607927c7757af17cd6fce0e545e5c48584 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 7 Sep 2016 13:36:08 -0400 Subject: [PATCH 129/871] Doc: minor documentation improvements about extensions. Document the formerly-undocumented behavior that schema and comment control-file entries for an extension are honored only during initial installation, whereas other properties are also honored during updates. While at it, do some copy-editing on the recently-added docs for CREATE EXTENSION ... CASCADE, use links for some formerly vague cross references, and make a couple other minor improvements. Back-patch to 9.6 where CASCADE was added. The other parts of this could go further back, but they're probably not important enough to bother. --- doc/src/sgml/extend.sgml | 32 +++++++++------ doc/src/sgml/ref/create_extension.sgml | 54 +++++++++++--------------- 2 files changed, 42 insertions(+), 44 deletions(-) diff --git a/doc/src/sgml/extend.sgml b/doc/src/sgml/extend.sgml index f050ff1f66..df88380a23 100644 --- a/doc/src/sgml/extend.sgml +++ b/doc/src/sgml/extend.sgml @@ -335,11 +335,13 @@ by pg_dump. Such a change is usually only sensible if you concurrently make the same change in the extension's script file. (But there are special provisions for tables containing configuration - data; see below.) + data; see .) + In production situations, it's generally better to create an extension + update script to perform changes to extension member objects. - The extension script may set privileges on objects which are part of the + The extension script may set privileges on objects that are part of the extension via GRANT and REVOKE statements. The final set of privileges for each object (if any are set) will be stored in the @@ -453,9 +455,11 @@ comment (string) - A comment (any string) about the extension. Alternatively, - the comment can be set by means of the - command in the script file. + A comment (any string) about the extension. The comment is applied + when initially creating an extension, but not during extension updates + (since that might override user-added comments). Alternatively, + the extension's comment can be set by writing + a command in the script file. @@ -518,7 +522,7 @@ its contained objects into a different schema after initial creation of the extension. The default is false, i.e. the extension is not relocatable. - See below for more information. + See for more information. @@ -529,7 +533,10 @@ This parameter can only be set for non-relocatable extensions. It forces the extension to be loaded into exactly the named schema - and not any other. See below for more information. + and not any other. + The schema parameter is consulted only when + initially creating an extension, not during extension updates. + See for more information. @@ -562,7 +569,8 @@ comments) by the extension mechanism. This provision is commonly used to throw an error if the script file is fed to psql rather than being loaded via CREATE EXTENSION (see example - script below). Without that, users might accidentally load the + script in ). + Without that, users might accidentally load the extension's contents as loose objects rather than as an extension, a state of affairs that's a bit tedious to recover from. @@ -580,7 +588,7 @@ - + Extension Relocatability @@ -678,7 +686,7 @@ SET LOCAL search_path TO @extschema@; - + Extension Configuration Tables @@ -762,7 +770,7 @@ SELECT pg_catalog.pg_extension_config_dump('my_config', 'WHERE NOT standard_entr out but the dump will not be able to be restored directly and user intervention will be required. - + Sequences associated with serial or bigserial columns need to be directly marked to dump their state. Marking their parent @@ -877,7 +885,7 @@ SELECT * FROM pg_extension_update_paths('extension_name'); - + Extension Example diff --git a/doc/src/sgml/ref/create_extension.sgml b/doc/src/sgml/ref/create_extension.sgml index 007d8c9330..14e910115a 100644 --- a/doc/src/sgml/ref/create_extension.sgml +++ b/doc/src/sgml/ref/create_extension.sgml @@ -95,35 +95,21 @@ CREATE EXTENSION [ IF NOT EXISTS ] extension_name If not specified, and the extension's control file does not specify a schema either, the current default object creation schema is used. + - If the extension specifies schema in its control file, - the schema cannot be overridden with SCHEMA clause. - The SCHEMA clause in this case works as follows: - - - - If schema_name matches - the schema in control file, it will be used normally as there is no - conflict. - - - - - If the CASCADE clause is given, the - schema_name will only - be used for the missing required extensions which do not specify - schema in their control files. - - - - - If schema_name is not - the same as the one in extension's control file and the - CASCADE clause is not given, error will be thrown. - - - + If the extension specifies a schema parameter in its + control file, then that schema cannot be overridden with + a SCHEMA clause. Normally, an error will be raised if + a SCHEMA clause is given and it conflicts with the + extension's schema parameter. However, if + the CASCADE clause is also given, + then schema_name is + ignored when it conflicts. The + given schema_name will be + used for installation of any needed extensions that do not + specify schema in their control files. + Remember that the extension itself is not considered to be within any schema: extensions have unqualified names that must be unique @@ -147,7 +133,8 @@ CREATE EXTENSION [ IF NOT EXISTS ] extension_name old_version - FROM old_version + + FROM old_version must be specified when, and only when, you are attempting to install an extension that replaces an old style module that is just a collection of objects not packaged into an extension. This option @@ -174,10 +161,13 @@ CREATE EXTENSION [ IF NOT EXISTS ] extension_name CASCADE - Try to install extension including the required dependencies - recursively. The SCHEMA option will be propagated - to the required extensions. Other options are not recursively - applied when using this clause. + Automatically install any extensions that this extension depends on + that are not already installed. Their dependencies are likewise + automatically installed, recursively. The SCHEMA clause, + if given, applies to all extensions that get installed this way. + Other options of the statement are not applied to + automatically-installed extensions; in particular, their default + versions are always selected. From 0ab9c56d0fe3acc9d4717a9cbac6ef3369275b90 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 7 Sep 2016 16:11:56 -0400 Subject: [PATCH 130/871] Support renaming an existing value of an enum type. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not much to be said about this patch: it does what it says on the tin. In passing, rename AlterEnumStmt.skipIfExists to skipIfNewValExists to clarify what it actually does. In the discussion of this patch we considered supporting other similar options, such as IF EXISTS on the type as a whole or IF NOT EXISTS on the target name. This patch doesn't actually add any such feature, but it might happen later. Dagfinn Ilmari Mannsåker, reviewed by Emre Hasegeli Discussion: --- doc/src/sgml/ref/alter_type.sgml | 68 ++++++++++++++++++------ src/backend/catalog/pg_enum.c | 85 ++++++++++++++++++++++++++++++ src/backend/commands/typecmds.c | 20 ++++--- src/backend/nodes/copyfuncs.c | 3 +- src/backend/nodes/equalfuncs.c | 3 +- src/backend/parser/gram.y | 20 +++++-- src/include/catalog/pg_enum.h | 2 + src/include/nodes/parsenodes.h | 3 +- src/test/regress/expected/enum.out | 22 ++++++++ src/test/regress/sql/enum.sql | 11 ++++ 10 files changed, 208 insertions(+), 29 deletions(-) diff --git a/doc/src/sgml/ref/alter_type.sgml b/doc/src/sgml/ref/alter_type.sgml index aec73f6285..fdb4f3367d 100644 --- a/doc/src/sgml/ref/alter_type.sgml +++ b/doc/src/sgml/ref/alter_type.sgml @@ -28,7 +28,8 @@ ALTER TYPE name OWNER TO { name RENAME ATTRIBUTE attribute_name TO new_attribute_name [ CASCADE | RESTRICT ] ALTER TYPE name RENAME TO new_name ALTER TYPE name SET SCHEMA new_schema -ALTER TYPE name ADD VALUE [ IF NOT EXISTS ] new_enum_value [ { BEFORE | AFTER } existing_enum_value ] +ALTER TYPE name ADD VALUE [ IF NOT EXISTS ] new_enum_value [ { BEFORE | AFTER } neighbor_enum_value ] +ALTER TYPE name RENAME VALUE existing_enum_value TO new_enum_value where action is one of: @@ -124,21 +125,13 @@ ALTER TYPE name ADD VALUE [ IF NOT - CASCADE + RENAME VALUE - Automatically propagate the operation to typed tables of the - type being altered, and their descendants. - - - - - - RESTRICT - - - Refuse the operation if the type being altered is the type of a - typed table. This is the default. + This form renames a value of an enum type. + The value's place in the enum's ordering is not affected. + An error will occur if the specified value is not present or the new + name is already present. @@ -241,14 +234,15 @@ ALTER TYPE name ADD VALUE [ IF NOT new_enum_value - The new value to be added to an enum type's list of values. + The new value to be added to an enum type's list of values, + or the new name to be given to an existing value. Like all enum literals, it needs to be quoted. - existing_enum_value + neighbor_enum_value The existing enum value that the new value should be added immediately @@ -258,6 +252,36 @@ ALTER TYPE name ADD VALUE [ IF NOT + + existing_enum_value + + + The existing enum value that should be renamed. + Like all enum literals, it needs to be quoted. + + + + + + CASCADE + + + Automatically propagate the operation to typed tables of the + type being altered, and their descendants. + + + + + + RESTRICT + + + Refuse the operation if the type being altered is the type of a + typed table. This is the default. + + + + @@ -270,6 +294,8 @@ ALTER TYPE name ADD VALUE [ IF NOT an enum type) is executed inside a transaction block, the new value cannot be used until after the transaction has been committed, except in the case that the enum type itself was created earlier in the same transaction. + Likewise, when a pre-existing enum value is renamed, the transaction must + be committed before the renamed value can be used. @@ -323,7 +349,15 @@ ALTER TYPE compfoo ADD ATTRIBUTE f3 int; To add a new value to an enum type in a particular sort position: ALTER TYPE colors ADD VALUE 'orange' AFTER 'red'; - + + + + + To rename an enum value: + +ALTER TYPE colors RENAME VALUE 'purple' TO 'mauve'; + + diff --git a/src/backend/catalog/pg_enum.c b/src/backend/catalog/pg_enum.c index c66f9632c2..1f0ffcfa15 100644 --- a/src/backend/catalog/pg_enum.c +++ b/src/backend/catalog/pg_enum.c @@ -465,6 +465,91 @@ AddEnumLabel(Oid enumTypeOid, } +/* + * RenameEnumLabel + * Rename a label in an enum set. + */ +void +RenameEnumLabel(Oid enumTypeOid, + const char *oldVal, + const char *newVal) +{ + Relation pg_enum; + HeapTuple enum_tup; + Form_pg_enum en; + CatCList *list; + int nelems; + HeapTuple old_tup; + bool found_new; + int i; + + /* check length of new label is ok */ + if (strlen(newVal) > (NAMEDATALEN - 1)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_NAME), + errmsg("invalid enum label \"%s\"", newVal), + errdetail("Labels must be %d characters or less.", + NAMEDATALEN - 1))); + + /* + * Acquire a lock on the enum type, which we won't release until commit. + * This ensures that two backends aren't concurrently modifying the same + * enum type. Since we are not changing the type's sort order, this is + * probably not really necessary, but there seems no reason not to take + * the lock to be sure. + */ + LockDatabaseObject(TypeRelationId, enumTypeOid, 0, ExclusiveLock); + + pg_enum = heap_open(EnumRelationId, RowExclusiveLock); + + /* Get the list of existing members of the enum */ + list = SearchSysCacheList1(ENUMTYPOIDNAME, + ObjectIdGetDatum(enumTypeOid)); + nelems = list->n_members; + + /* + * Locate the element to rename and check if the new label is already in + * use. (The unique index on pg_enum would catch that anyway, but we + * prefer a friendlier error message.) + */ + old_tup = NULL; + found_new = false; + for (i = 0; i < nelems; i++) + { + enum_tup = &(list->members[i]->tuple); + en = (Form_pg_enum) GETSTRUCT(enum_tup); + if (strcmp(NameStr(en->enumlabel), oldVal) == 0) + old_tup = enum_tup; + if (strcmp(NameStr(en->enumlabel), newVal) == 0) + found_new = true; + } + if (!old_tup) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" is not an existing enum label", + oldVal))); + if (found_new) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("enum label \"%s\" already exists", + newVal))); + + /* OK, make a writable copy of old tuple */ + enum_tup = heap_copytuple(old_tup); + en = (Form_pg_enum) GETSTRUCT(enum_tup); + + ReleaseCatCacheList(list); + + /* Update the pg_enum entry */ + namestrcpy(&en->enumlabel, newVal); + simple_heap_update(pg_enum, &enum_tup->t_self, enum_tup); + CatalogUpdateIndexes(pg_enum, enum_tup); + heap_freetuple(enum_tup); + + heap_close(pg_enum, RowExclusiveLock); +} + + /* * RenumberEnumType * Renumber existing enum elements to have sort positions 1..n. diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c index 6cc7106467..41fd2dae7f 100644 --- a/src/backend/commands/typecmds.c +++ b/src/backend/commands/typecmds.c @@ -1241,17 +1241,25 @@ AlterEnum(AlterEnumStmt *stmt) /* Check it's an enum and check user has permission to ALTER the enum */ checkEnumOwner(tup); - /* Add the new label */ - AddEnumLabel(enum_type_oid, stmt->newVal, - stmt->newValNeighbor, stmt->newValIsAfter, - stmt->skipIfExists); + ReleaseSysCache(tup); + + if (stmt->oldVal) + { + /* Rename an existing label */ + RenameEnumLabel(enum_type_oid, stmt->oldVal, stmt->newVal); + } + else + { + /* Add a new label */ + AddEnumLabel(enum_type_oid, stmt->newVal, + stmt->newValNeighbor, stmt->newValIsAfter, + stmt->skipIfNewValExists); + } InvokeObjectPostAlterHook(TypeRelationId, enum_type_oid, 0); ObjectAddressSet(address, TypeRelationId, enum_type_oid); - ReleaseSysCache(tup); - return address; } diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index be2207e318..4f39dad66b 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3375,10 +3375,11 @@ _copyAlterEnumStmt(const AlterEnumStmt *from) AlterEnumStmt *newnode = makeNode(AlterEnumStmt); COPY_NODE_FIELD(typeName); + COPY_STRING_FIELD(oldVal); COPY_STRING_FIELD(newVal); COPY_STRING_FIELD(newValNeighbor); COPY_SCALAR_FIELD(newValIsAfter); - COPY_SCALAR_FIELD(skipIfExists); + COPY_SCALAR_FIELD(skipIfNewValExists); return newnode; } diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index c4ec4077a6..4800165a91 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1465,10 +1465,11 @@ static bool _equalAlterEnumStmt(const AlterEnumStmt *a, const AlterEnumStmt *b) { COMPARE_NODE_FIELD(typeName); + COMPARE_STRING_FIELD(oldVal); COMPARE_STRING_FIELD(newVal); COMPARE_STRING_FIELD(newValNeighbor); COMPARE_SCALAR_FIELD(newValIsAfter); - COMPARE_SCALAR_FIELD(skipIfExists); + COMPARE_SCALAR_FIELD(skipIfNewValExists); return true; } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index b69a77a588..1526c73a1c 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -5257,30 +5257,44 @@ AlterEnumStmt: { AlterEnumStmt *n = makeNode(AlterEnumStmt); n->typeName = $3; + n->oldVal = NULL; n->newVal = $7; n->newValNeighbor = NULL; n->newValIsAfter = true; - n->skipIfExists = $6; + n->skipIfNewValExists = $6; $$ = (Node *) n; } | ALTER TYPE_P any_name ADD_P VALUE_P opt_if_not_exists Sconst BEFORE Sconst { AlterEnumStmt *n = makeNode(AlterEnumStmt); n->typeName = $3; + n->oldVal = NULL; n->newVal = $7; n->newValNeighbor = $9; n->newValIsAfter = false; - n->skipIfExists = $6; + n->skipIfNewValExists = $6; $$ = (Node *) n; } | ALTER TYPE_P any_name ADD_P VALUE_P opt_if_not_exists Sconst AFTER Sconst { AlterEnumStmt *n = makeNode(AlterEnumStmt); n->typeName = $3; + n->oldVal = NULL; n->newVal = $7; n->newValNeighbor = $9; n->newValIsAfter = true; - n->skipIfExists = $6; + n->skipIfNewValExists = $6; + $$ = (Node *) n; + } + | ALTER TYPE_P any_name RENAME VALUE_P Sconst TO Sconst + { + AlterEnumStmt *n = makeNode(AlterEnumStmt); + n->typeName = $3; + n->oldVal = $6; + n->newVal = $8; + n->newValNeighbor = NULL; + n->newValIsAfter = false; + n->skipIfNewValExists = false; $$ = (Node *) n; } ; diff --git a/src/include/catalog/pg_enum.h b/src/include/catalog/pg_enum.h index dd32443b91..901d3adbb9 100644 --- a/src/include/catalog/pg_enum.h +++ b/src/include/catalog/pg_enum.h @@ -67,5 +67,7 @@ extern void EnumValuesDelete(Oid enumTypeOid); extern void AddEnumLabel(Oid enumTypeOid, const char *newVal, const char *neighbor, bool newValIsAfter, bool skipIfExists); +extern void RenameEnumLabel(Oid enumTypeOid, + const char *oldVal, const char *newVal); #endif /* PG_ENUM_H */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 3716c2eef9..8d3dcf4d4c 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2708,10 +2708,11 @@ typedef struct AlterEnumStmt { NodeTag type; List *typeName; /* qualified name (list of Value strings) */ + char *oldVal; /* old enum value's name, if renaming */ char *newVal; /* new enum value's name */ char *newValNeighbor; /* neighboring enum value, if specified */ bool newValIsAfter; /* place new enum value after neighbor? */ - bool skipIfExists; /* no error if label already exists */ + bool skipIfNewValExists; /* no error if new already exists? */ } AlterEnumStmt; /* ---------------------- diff --git a/src/test/regress/expected/enum.out b/src/test/regress/expected/enum.out index d4a45a306b..514d1d01a1 100644 --- a/src/test/regress/expected/enum.out +++ b/src/test/regress/expected/enum.out @@ -556,6 +556,28 @@ CREATE TABLE enumtest_bogus_child(parent bogus REFERENCES enumtest_parent); ERROR: foreign key constraint "enumtest_bogus_child_parent_fkey" cannot be implemented DETAIL: Key columns "parent" and "id" are of incompatible types: bogus and rainbow. DROP TYPE bogus; +-- check renaming a value +ALTER TYPE rainbow RENAME VALUE 'red' TO 'crimson'; +SELECT enumlabel, enumsortorder +FROM pg_enum +WHERE enumtypid = 'rainbow'::regtype +ORDER BY 2; + enumlabel | enumsortorder +-----------+--------------- + crimson | 1 + orange | 2 + yellow | 3 + green | 4 + blue | 5 + purple | 6 +(6 rows) + +-- check that renaming a non-existent value fails +ALTER TYPE rainbow RENAME VALUE 'red' TO 'crimson'; +ERROR: "red" is not an existing enum label +-- check that renaming to an existent value fails +ALTER TYPE rainbow RENAME VALUE 'blue' TO 'green'; +ERROR: enum label "green" already exists -- -- check transactional behaviour of ALTER TYPE ... ADD VALUE -- diff --git a/src/test/regress/sql/enum.sql b/src/test/regress/sql/enum.sql index d25e8dedb6..d7e87143a0 100644 --- a/src/test/regress/sql/enum.sql +++ b/src/test/regress/sql/enum.sql @@ -257,6 +257,17 @@ CREATE TYPE bogus AS ENUM('good', 'bad', 'ugly'); CREATE TABLE enumtest_bogus_child(parent bogus REFERENCES enumtest_parent); DROP TYPE bogus; +-- check renaming a value +ALTER TYPE rainbow RENAME VALUE 'red' TO 'crimson'; +SELECT enumlabel, enumsortorder +FROM pg_enum +WHERE enumtypid = 'rainbow'::regtype +ORDER BY 2; +-- check that renaming a non-existent value fails +ALTER TYPE rainbow RENAME VALUE 'red' TO 'crimson'; +-- check that renaming to an existent value fails +ALTER TYPE rainbow RENAME VALUE 'blue' TO 'green'; + -- -- check transactional behaviour of ALTER TYPE ... ADD VALUE -- From c9cf432ef32a9d29323b9b079178c1a6be126ff8 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Wed, 7 Sep 2016 20:51:28 -0400 Subject: [PATCH 131/871] 9.6 release notes: correct summary item about freeze Previously it less precisely talked about autovacuum. Backpatch-through: 9.6 --- doc/src/sgml/release-9.6.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 578c3d1fdb..ddd280c85a 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -29,7 +29,7 @@ - Autovacuum no longer performs repetitive scanning of old data + Avoid scanning pages unnecessarily during vacuum freeze operations From 976a9bbd0251ea112898f85314646801e7e6207d Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Thu, 8 Sep 2016 01:40:53 -0400 Subject: [PATCH 132/871] MSVC: Place gendef.pl temporary file in the target directory. Until now, it used the current working directory. This makes it safe for simultaneous invocations of gendef.pl, with different target directories, to run from a single current working directory, such as $(top_srcdir). The MSVC build system will soon rely on this. Christian Ullrich, reviewed by Michael Paquier. --- src/tools/msvc/gendef.pl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/tools/msvc/gendef.pl b/src/tools/msvc/gendef.pl index 8ccaab3551..a6c43c2c39 100644 --- a/src/tools/msvc/gendef.pl +++ b/src/tools/msvc/gendef.pl @@ -3,6 +3,7 @@ use warnings; use strict; use 5.8.0; +use File::Spec::Functions qw(splitpath catpath); use List::Util qw(max); # @@ -14,9 +15,11 @@ sub dumpsyms { my ($objfile, $symfile) = @_; - system("dumpbin /symbols /out:symbols.out $_ >NUL") + my ($symvol, $symdirs, $symbase) = splitpath($symfile); + my $tmpfile = catpath($symvol, $symdirs, "symbols.out"); + system("dumpbin /symbols /out:$tmpfile $_ >NUL") && die "Could not call dumpbin"; - rename("symbols.out", $symfile); + rename($tmpfile, $symfile); } # Given a symbol file path, loops over its contents From d299eb41dfc7b73dec80f22554b952f01c9d54a4 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Thu, 8 Sep 2016 01:42:09 -0400 Subject: [PATCH 133/871] MSVC: Pass any user-set MSBFLAGS to MSBuild and VCBUILD. This is particularly useful to pass /m, to perform a parallel build. Christian Ullrich, reviewed by Michael Paquier. --- doc/src/sgml/install-windows.sgml | 8 ++++++++ src/tools/msvc/build.pl | 7 ++++--- src/tools/msvc/clean.bat | 2 +- src/tools/msvc/vcregress.pl | 3 ++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index 8cd189c8e1..50116f315d 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -145,6 +145,14 @@ $ENV{PATH}=$ENV{PATH} . ';c:\some\where\bison\bin'; + + To pass additional command line arguments to the Visual Studio build + command (msbuild or vcbuild): + +$ENV{MSBFLAGS}="/m"; + + + Requirements diff --git a/src/tools/msvc/build.pl b/src/tools/msvc/build.pl index 007e3c73b2..52739774c7 100644 --- a/src/tools/msvc/build.pl +++ b/src/tools/msvc/build.pl @@ -38,6 +38,7 @@ BEGIN # check what sort of build we are doing my $bconf = $ENV{CONFIG} || "Release"; +my $msbflags = $ENV{MSBFLAGS} || ""; my $buildwhat = $ARGV[1] || ""; if (uc($ARGV[0]) eq 'DEBUG') { @@ -53,16 +54,16 @@ BEGIN if ($buildwhat and $vcver >= 10.00) { system( - "msbuild $buildwhat.vcxproj /verbosity:normal /p:Configuration=$bconf" + "msbuild $buildwhat.vcxproj $msbflags /verbosity:normal /p:Configuration=$bconf" ); } elsif ($buildwhat) { - system("vcbuild $buildwhat.vcproj $bconf"); + system("vcbuild $msbflags $buildwhat.vcproj $bconf"); } else { - system("msbuild pgsql.sln /verbosity:normal /p:Configuration=$bconf"); + system("msbuild pgsql.sln $msbflags /verbosity:normal /p:Configuration=$bconf"); } # report status diff --git a/src/tools/msvc/clean.bat b/src/tools/msvc/clean.bat index 469b8a24b2..e21e37f6f7 100755 --- a/src/tools/msvc/clean.bat +++ b/src/tools/msvc/clean.bat @@ -107,6 +107,6 @@ REM for /r %%f in (*.sql) do if exist %%f.in del %%f cd %D% REM Clean up ecpg regression test files -msbuild /NoLogo ecpg_regression.proj /t:clean /v:q +msbuild %MSBFLAGS% /NoLogo ecpg_regression.proj /t:clean /v:q goto :eof diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl index b4f946474f..bcf22677ac 100644 --- a/src/tools/msvc/vcregress.pl +++ b/src/tools/msvc/vcregress.pl @@ -138,8 +138,9 @@ sub check sub ecpgcheck { + my $msbflags = $ENV{MSBFLAGS} || ""; chdir $startdir; - system("msbuild ecpg_regression.proj /p:config=$Config"); + system("msbuild ecpg_regression.proj $msbflags /p:config=$Config"); my $status = $? >> 8; exit $status if $status; InstallTemp(); From 67c6bd1ca3ce75778138bf4713444a5a6b46032e Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Thu, 8 Sep 2016 10:32:58 +0100 Subject: [PATCH 134/871] Fix minor memory leak in Standby startup StandbyRecoverPreparedTransactions() leaked the buffer used for two phase state file. This was leaked once at startup and at every shutdown checkpoint seen. Backpatch to 9.6 Stas Kelvich --- src/backend/access/transam/twophase.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 9f55adcaf5..1323fb508d 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1886,6 +1886,8 @@ StandbyRecoverPreparedTransactions(bool overwriteOK) Assert(TransactionIdFollows(subxid, xid)); SubTransSetParent(xid, subxid, overwriteOK); } + + pfree(buf); } } FreeDir(cldir); From e97e9c57bd22b2dfbfaf41f7d5c69789f7fad554 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 8 Sep 2016 10:48:03 -0400 Subject: [PATCH 135/871] Don't print database's tablespace in pg_dump -C --no-tablespaces output. If the database has a non-default tablespace, we emitted a TABLESPACE clause in the CREATE DATABASE command emitted by -C, even if --no-tablespaces was also specified. This seems wrong, and it's inconsistent with what pg_dumpall does, so change it. Per bug #14315 from Danylo Hlynskyi. Back-patch to 9.5. The bug is much older, but it'd be a more invasive change before 9.5 because dumpDatabase() hasn't got an easy way to get to the outputNoTablespaces flag. Doesn't seem worth the work given the lack of previous complaints. Report: <20160908081953.1402.75347@wrigleys.postgresql.org> --- src/bin/pg_dump/pg_dump.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index a5c2d09e29..165200f0fc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -2552,7 +2552,8 @@ dumpDatabase(Archive *fout) appendPQExpBufferStr(creaQry, " LC_CTYPE = "); appendStringLiteralAH(creaQry, ctype, fout); } - if (strlen(tablespace) > 0 && strcmp(tablespace, "pg_default") != 0) + if (strlen(tablespace) > 0 && strcmp(tablespace, "pg_default") != 0 && + !dopt->outputNoTablespaces) appendPQExpBuffer(creaQry, " TABLESPACE = %s", fmtId(tablespace)); appendPQExpBufferStr(creaQry, ";\n"); From df5d9bb8d5074138e6fea63ac8acd9b95a0eb859 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 8 Sep 2016 13:12:01 -0400 Subject: [PATCH 136/871] Allow pg_dump to dump non-extension members of an extension-owned schema. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, if a schema was created by an extension, a normal pg_dump run (not --binary-upgrade) would summarily skip every object in that schema. In a case where an extension creates a schema and then users create other objects within that schema, this does the wrong thing: we want pg_dump to skip the schema but still create the non-extension-owned objects. There's no easy way to fix this pre-9.6, because in earlier versions the "dump" status for a schema is just a bool and there's no way to distinguish "dump me" from "dump my members". However, as of 9.6 we do have enough state to represent that, so this is a simple correction of the logic in selectDumpableNamespace. In passing, make some cosmetic fixes in nearby code. Martín Marqués, reviewed by Michael Paquier Discussion: <99581032-71de-6466-c325-069861f1947d@2ndquadrant.com> --- src/bin/pg_dump/pg_dump.c | 28 ++- src/test/modules/test_pg_dump/t/001_base.pl | 206 +++++++++++++++++- .../test_pg_dump/test_pg_dump--1.0.sql | 24 ++ 3 files changed, 247 insertions(+), 11 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 165200f0fc..ba9c276593 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -1302,7 +1302,7 @@ checkExtensionMembership(DumpableObject *dobj, Archive *fout) /* * In 9.6 and above, mark the member object to have any non-initial ACL, - * policies, and security lables dumped. + * policies, and security labels dumped. * * Note that any initial ACLs (see pg_init_privs) will be removed when we * extract the information about the object. We don't provide support for @@ -1324,8 +1324,8 @@ checkExtensionMembership(DumpableObject *dobj, Archive *fout) dobj->dump = DUMP_COMPONENT_NONE; else dobj->dump = ext->dobj.dump_contains & (DUMP_COMPONENT_ACL | - DUMP_COMPONENT_SECLABEL | DUMP_COMPONENT_POLICY); - + DUMP_COMPONENT_SECLABEL | + DUMP_COMPONENT_POLICY); } return true; @@ -1338,15 +1338,11 @@ checkExtensionMembership(DumpableObject *dobj, Archive *fout) static void selectDumpableNamespace(NamespaceInfo *nsinfo, Archive *fout) { - if (checkExtensionMembership(&nsinfo->dobj, fout)) - return; /* extension membership overrides all else */ - /* * If specific tables are being dumped, do not dump any complete * namespaces. If specific namespaces are being dumped, dump just those * namespaces. Otherwise, dump all non-system namespaces. */ - if (table_include_oids.head != NULL) nsinfo->dobj.dump_contains = nsinfo->dobj.dump = DUMP_COMPONENT_NONE; else if (schema_include_oids.head != NULL) @@ -1355,18 +1351,21 @@ selectDumpableNamespace(NamespaceInfo *nsinfo, Archive *fout) nsinfo->dobj.catId.oid) ? DUMP_COMPONENT_ALL : DUMP_COMPONENT_NONE; else if (fout->remoteVersion >= 90600 && - strncmp(nsinfo->dobj.name, "pg_catalog", - strlen("pg_catalog")) == 0) - + strcmp(nsinfo->dobj.name, "pg_catalog") == 0) + { /* * In 9.6 and above, we dump out any ACLs defined in pg_catalog, if * they are interesting (and not the original ACLs which were set at * initdb time, see pg_init_privs). */ nsinfo->dobj.dump_contains = nsinfo->dobj.dump = DUMP_COMPONENT_ACL; + } else if (strncmp(nsinfo->dobj.name, "pg_", 3) == 0 || strcmp(nsinfo->dobj.name, "information_schema") == 0) + { + /* Other system schemas don't get dumped */ nsinfo->dobj.dump_contains = nsinfo->dobj.dump = DUMP_COMPONENT_NONE; + } else nsinfo->dobj.dump_contains = nsinfo->dobj.dump = DUMP_COMPONENT_ALL; @@ -1377,6 +1376,15 @@ selectDumpableNamespace(NamespaceInfo *nsinfo, Archive *fout) simple_oid_list_member(&schema_exclude_oids, nsinfo->dobj.catId.oid)) nsinfo->dobj.dump_contains = nsinfo->dobj.dump = DUMP_COMPONENT_NONE; + + /* + * If the schema belongs to an extension, allow extension membership to + * override the dump decision for the schema itself. However, this does + * not change dump_contains, so this won't change what we do with objects + * within the schema. (If they belong to the extension, they'll get + * suppressed by it, otherwise not.) + */ + (void) checkExtensionMembership(&nsinfo->dobj, fout); } /* diff --git a/src/test/modules/test_pg_dump/t/001_base.pl b/src/test/modules/test_pg_dump/t/001_base.pl index f02beb3d9c..55f0eb4547 100644 --- a/src/test/modules/test_pg_dump/t/001_base.pl +++ b/src/test/modules/test_pg_dump/t/001_base.pl @@ -429,7 +429,211 @@ unlike => { no_privs => 1, pg_dumpall_globals => 1, - section_post_data => 1, }, },); + section_post_data => 1, }, }, + # Objects included in extension part of a schema created by this extension */ + 'CREATE TABLE regress_pg_dump_schema.test_table' => { + regexp => qr/^ + \QCREATE TABLE test_table (\E + \n\s+\Qcol1 integer,\E + \n\s+\Qcol2 integer\E + \n\);$/xm, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_privs => 1, + no_owner => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + 'GRANT SELECT ON regress_pg_dump_schema.test_table' => { + regexp => qr/^ + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n + \QGRANT SELECT ON TABLE test_table TO regress_dump_test_role;\E\n + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E + $/xms, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_owner => 1, + no_privs => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + 'CREATE SEQUENCE regress_pg_dump_schema.test_seq' => { + regexp => qr/^ + \QCREATE SEQUENCE test_seq\E + \n\s+\QSTART WITH 1\E + \n\s+\QINCREMENT BY 1\E + \n\s+\QNO MINVALUE\E + \n\s+\QNO MAXVALUE\E + \n\s+\QCACHE 1;\E + $/xm, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_privs => 1, + no_owner => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + 'GRANT USAGE ON regress_pg_dump_schema.test_seq' => { + regexp => qr/^ + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n + \QGRANT USAGE ON SEQUENCE test_seq TO regress_dump_test_role;\E\n + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E + $/xms, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_owner => 1, + no_privs => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + 'CREATE TYPE regress_pg_dump_schema.test_type' => { + regexp => qr/^ + \QCREATE TYPE test_type AS (\E + \n\s+\Qcol1 integer\E + \n\);$/xm, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_privs => 1, + no_owner => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + 'GRANT USAGE ON regress_pg_dump_schema.test_type' => { + regexp => qr/^ + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n + \QGRANT ALL ON TYPE test_type TO regress_dump_test_role;\E\n + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E + $/xms, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_owner => 1, + no_privs => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + 'CREATE FUNCTION regress_pg_dump_schema.test_func' => { + regexp => qr/^ + \QCREATE FUNCTION test_func() RETURNS integer\E + \n\s+\QLANGUAGE sql\E + $/xm, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_privs => 1, + no_owner => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + 'GRANT ALL ON regress_pg_dump_schema.test_func' => { + regexp => qr/^ + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n + \QGRANT ALL ON FUNCTION test_func() TO regress_dump_test_role;\E\n + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E + $/xms, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_owner => 1, + no_privs => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + 'CREATE AGGREGATE regress_pg_dump_schema.test_agg' => { + regexp => qr/^ + \QCREATE AGGREGATE test_agg(smallint) (\E + \n\s+\QSFUNC = int2_sum,\E + \n\s+\QSTYPE = bigint\E + \n\);$/xm, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_privs => 1, + no_owner => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + 'GRANT ALL ON regress_pg_dump_schema.test_agg' => { + regexp => qr/^ + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n + \QGRANT ALL ON FUNCTION test_agg(smallint) TO regress_dump_test_role;\E\n + \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E + $/xms, + like => { binary_upgrade => 1, }, + unlike => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_owner => 1, + no_privs => 1, + pg_dumpall_globals => 1, + schema_only => 1, + section_pre_data => 1, + section_post_data => 1, }, }, + # Objects not included in extension, part of schema created by extension + 'CREATE TABLE regress_pg_dump_schema.external_tab' => { + create_order => 4, + create_sql => 'CREATE TABLE regress_pg_dump_schema.external_tab + (col1 int);', + regexp => qr/^ + \QCREATE TABLE external_tab (\E + \n\s+\Qcol1 integer\E + \n\);$/xm, + like => { + binary_upgrade => 1, + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + no_owner => 1, + no_privs => 1, + schema_only => 1, + section_pre_data => 1, }, + unlike => { + pg_dumpall_globals => 1, + section_post_data => 1, }, }, ); ######################################### # Create a PG instance to test actually dumping from diff --git a/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql b/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql index c2fe90d5ab..ca9fb18e54 100644 --- a/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql +++ b/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql @@ -10,6 +10,8 @@ CREATE TABLE regress_pg_dump_table ( CREATE SEQUENCE regress_pg_dump_seq; +CREATE SCHEMA regress_pg_dump_schema; + GRANT USAGE ON regress_pg_dump_seq TO regress_dump_test_role; GRANT SELECT ON regress_pg_dump_table TO regress_dump_test_role; @@ -19,3 +21,25 @@ GRANT SELECT(col2) ON regress_pg_dump_table TO regress_dump_test_role; REVOKE SELECT(col2) ON regress_pg_dump_table FROM regress_dump_test_role; CREATE ACCESS METHOD regress_test_am TYPE INDEX HANDLER bthandler; + +-- Create a set of objects that are part of the schema created by +-- this extension. +CREATE TABLE regress_pg_dump_schema.test_table ( + col1 int, + col2 int +); +GRANT SELECT ON regress_pg_dump_schema.test_table TO regress_dump_test_role; + +CREATE SEQUENCE regress_pg_dump_schema.test_seq; +GRANT USAGE ON regress_pg_dump_schema.test_seq TO regress_dump_test_role; + +CREATE TYPE regress_pg_dump_schema.test_type AS (col1 int); +GRANT USAGE ON TYPE regress_pg_dump_schema.test_type TO regress_dump_test_role; + +CREATE FUNCTION regress_pg_dump_schema.test_func () RETURNS int +AS 'SELECT 1;' LANGUAGE SQL; +GRANT EXECUTE ON FUNCTION regress_pg_dump_schema.test_func() TO regress_dump_test_role; + +CREATE AGGREGATE regress_pg_dump_schema.test_agg(int2) +(SFUNC = int2_sum, STYPE = int8); +GRANT EXECUTE ON FUNCTION regress_pg_dump_schema.test_agg(int2) TO regress_dump_test_role; From 19acee8c5adb68b96222e41c084efbc9b31d397a Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 8 Sep 2016 14:39:05 -0300 Subject: [PATCH 137/871] Fix two src/test/modules Makefiles commit_ts and test_pg_dump were declaring targets before including the PGXS stanza, which meant that the "all" target customarily defined as the first (and therefore default target) was not the default anymore. Fix that by moving those target definitions to after PGXS. commit_ts was initially good, but I broke it in commit 9def031bd2; test_pg_dump was born broken, probably copying from commit_ts' mistake. In passing, fix a comment mistake in test_pg_dump/Makefile. Backpatch to 9.6. Noted by Tom Lane. --- src/test/modules/commit_ts/Makefile | 10 +++++----- src/test/modules/test_pg_dump/Makefile | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/modules/commit_ts/Makefile b/src/test/modules/commit_ts/Makefile index f814f767c4..86b93b5e76 100644 --- a/src/test/modules/commit_ts/Makefile +++ b/src/test/modules/commit_ts/Makefile @@ -3,11 +3,6 @@ REGRESS = commit_timestamp REGRESS_OPTS = --temp-config=$(top_srcdir)/src/test/modules/commit_ts/commit_ts.conf -check: prove-check - -prove-check: - $(prove_check) - ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -18,3 +13,8 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif + +check: prove-check + +prove-check: + $(prove_check) diff --git a/src/test/modules/test_pg_dump/Makefile b/src/test/modules/test_pg_dump/Makefile index 08d8903e72..5050572777 100644 --- a/src/test/modules/test_pg_dump/Makefile +++ b/src/test/modules/test_pg_dump/Makefile @@ -1,4 +1,4 @@ -# src/test/modules/test_rls_hooks/Makefile +# src/test/modules/test_pg_dump/Makefile MODULE = test_pg_dump PGFILEDESC = "test_pg_dump - Test pg_dump with an extension" @@ -8,11 +8,6 @@ DATA = test_pg_dump--1.0.sql REGRESS = test_pg_dump -check: prove-check - -prove-check: - $(prove_check) - ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -23,3 +18,8 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif + +check: prove-check + +prove-check: + $(prove_check) From 769fd9d8e06bf862334a0e04134a3d2c665e5e5b Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 8 Sep 2016 16:51:09 -0700 Subject: [PATCH 138/871] Fix mdtruncate() to close fd.c handle of deleted segments. mdtruncate() forgot to FileClose() a segment's mdfd_vfd, when deleting it. That lead to a fd.c handle to a truncated file being kept open until backend exit. The issue appears to have been introduced way back in 1a5c450f3024ac5, before that the handle was closed inside FileUnlink(). The impact of this bug is limited - only VACUUM and ON COMMIT TRUNCATE for temporary tables, truncate files in place (i.e. TRUNCATE itself is not affected), and the relation has to be bigger than 1GB. The consequences of a leaked fd.c handle aren't severe either. Discussion: <20160908220748.oqh37ukwqqncbl3n@alap3.anarazel.de> Backpatch: all supported releases --- src/backend/storage/smgr/md.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 1287142918..a94828b32c 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -976,6 +976,7 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) v = v->mdfd_chain; Assert(ov != reln->md_fd[forknum]); /* we never drop the 1st * segment */ + FileClose(ov->mdfd_vfd); pfree(ov); } else if (priorblocks + ((BlockNumber) RELSEG_SIZE) > nblocks) From 417fefaf089fc0b73607cbbe8bcd0bc9e89d08ef Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 8 Sep 2016 17:02:43 -0700 Subject: [PATCH 139/871] Faster PageIsVerified() for the all zeroes case. That's primarily useful for testing very large relations, using sparse files. Discussion: <20140331101001.GE13135@alap3.anarazel.de> Reviewed-By: Peter Geoghegan --- src/backend/storage/page/bufpage.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index f2a07f2111..1b70bfbe8c 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -81,7 +81,7 @@ bool PageIsVerified(Page page, BlockNumber blkno) { PageHeader p = (PageHeader) page; - char *pagebytes; + size_t *pagebytes; int i; bool checksum_failure = false; bool header_sane = false; @@ -118,10 +118,17 @@ PageIsVerified(Page page, BlockNumber blkno) return true; } - /* Check all-zeroes case */ + /* + * Check all-zeroes case. Luckily BLCKSZ is guaranteed to always be a + * multiple of size_t - and it's much faster to compare memory using the + * native word size. + */ + StaticAssertStmt(BLCKSZ == (BLCKSZ / sizeof(size_t)) * sizeof(size_t), + "BLCKSZ has to be a multiple of sizeof(size_t)"); + all_zeroes = true; - pagebytes = (char *) page; - for (i = 0; i < BLCKSZ; i++) + pagebytes = (size_t *) page; + for (i = 0; i < (BLCKSZ / sizeof(size_t)); i++) { if (pagebytes[i] != 0) { From 45e191e3aa62d47a8bc1a33f784286b2051f45cb Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 8 Sep 2016 17:02:43 -0700 Subject: [PATCH 140/871] Improve scalability of md.c for large relations. So far md.c used a linked list of segments. That proved to be a problem when processing large relations, because every smgr.c/md.c level access to a page incurred walking through a linked list of all preceding segments. Thus making accessing pages O(#segments). Replace the linked list of segments hanging off SMgrRelationData with an array of opened segments. That allows O(1) access to individual segments, if they've previously been opened. Discussion: <20140331101001.GE13135@alap3.anarazel.de> Reviewed-By: Peter Geoghegan, Tom Lane (in an older version) --- src/backend/storage/smgr/md.c | 393 ++++++++++++++++++-------------- src/backend/storage/smgr/smgr.c | 4 +- src/include/storage/smgr.h | 8 +- 3 files changed, 229 insertions(+), 176 deletions(-) diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index a94828b32c..1360f773df 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -92,27 +92,23 @@ * out to an unlinked old copy of a segment file that will eventually * disappear. * - * The file descriptor pointer (md_fd field) stored in the SMgrRelation - * cache is, therefore, just the head of a list of MdfdVec objects, one - * per segment. But note the md_fd pointer can be NULL, indicating - * relation not open. - * - * Also note that mdfd_chain == NULL does not necessarily mean the relation - * doesn't have another segment after this one; we may just not have - * opened the next segment yet. (We could not have "all segments are - * in the chain" as an invariant anyway, since another backend could - * extend the relation when we weren't looking.) We do not make chain + * File descriptors are stored in the per-fork md_seg_fds arrays inside + * SMgrRelation. The length of these arrays is stored in md_num_open_segs. + * Note that a fork's md_num_open_segs having a specific value does not + * necessarily mean the relation doesn't have additional segments; we may + * just not have opened the next segment yet. (We could not have "all + * segments are in the array" as an invariant anyway, since another backend + * could extend the relation while we aren't looking.) We do not have * entries for inactive segments, however; as soon as we find a partial * segment, we assume that any subsequent segments are inactive. * - * All MdfdVec objects are palloc'd in the MdCxt memory context. + * The entire MdfdVec array is palloc'd in the MdCxt memory context. */ typedef struct _MdfdVec { File mdfd_vfd; /* fd number in fd.c's pool */ BlockNumber mdfd_segno; /* segment number, from 0 */ - struct _MdfdVec *mdfd_chain; /* next segment, or NULL */ } MdfdVec; static MemoryContext MdCxt; /* context for all MdfdVec objects */ @@ -189,7 +185,9 @@ static MdfdVec *mdopen(SMgrRelation reln, ForkNumber forknum, int behavior); static void register_dirty_segment(SMgrRelation reln, ForkNumber forknum, MdfdVec *seg); static void register_unlink(RelFileNodeBackend rnode); -static MdfdVec *_fdvec_alloc(void); +static void _fdvec_resize(SMgrRelation reln, + ForkNumber forknum, + int nseg); static char *_mdfd_segpath(SMgrRelation reln, ForkNumber forknum, BlockNumber segno); static MdfdVec *_mdfd_openseg(SMgrRelation reln, ForkNumber forkno, @@ -294,13 +292,14 @@ mdexists(SMgrRelation reln, ForkNumber forkNum) void mdcreate(SMgrRelation reln, ForkNumber forkNum, bool isRedo) { + MdfdVec *mdfd; char *path; File fd; - if (isRedo && reln->md_fd[forkNum] != NULL) + if (isRedo && reln->md_num_open_segs[forkNum] > 0) return; /* created and opened already... */ - Assert(reln->md_fd[forkNum] == NULL); + Assert(reln->md_num_open_segs[forkNum] == 0); path = relpath(reln->smgr_rnode, forkNum); @@ -330,11 +329,10 @@ mdcreate(SMgrRelation reln, ForkNumber forkNum, bool isRedo) pfree(path); - reln->md_fd[forkNum] = _fdvec_alloc(); - - reln->md_fd[forkNum]->mdfd_vfd = fd; - reln->md_fd[forkNum]->mdfd_segno = 0; - reln->md_fd[forkNum]->mdfd_chain = NULL; + _fdvec_resize(reln, forkNum, 1); + mdfd = &reln->md_seg_fds[forkNum][0]; + mdfd->mdfd_vfd = fd; + mdfd->mdfd_segno = 0; } /* @@ -579,8 +577,8 @@ mdopen(SMgrRelation reln, ForkNumber forknum, int behavior) File fd; /* No work if already open */ - if (reln->md_fd[forknum]) - return reln->md_fd[forknum]; + if (reln->md_num_open_segs[forknum] > 0) + return &reln->md_seg_fds[forknum][0]; path = relpath(reln->smgr_rnode, forknum); @@ -612,11 +610,11 @@ mdopen(SMgrRelation reln, ForkNumber forknum, int behavior) pfree(path); - reln->md_fd[forknum] = mdfd = _fdvec_alloc(); - + _fdvec_resize(reln, forknum, 1); + mdfd = &reln->md_seg_fds[forknum][0]; mdfd->mdfd_vfd = fd; mdfd->mdfd_segno = 0; - mdfd->mdfd_chain = NULL; + Assert(_mdnblocks(reln, forknum, mdfd) <= ((BlockNumber) RELSEG_SIZE)); return mdfd; @@ -628,25 +626,29 @@ mdopen(SMgrRelation reln, ForkNumber forknum, int behavior) void mdclose(SMgrRelation reln, ForkNumber forknum) { - MdfdVec *v = reln->md_fd[forknum]; + int nopensegs = reln->md_num_open_segs[forknum]; /* No work if already closed */ - if (v == NULL) + if (nopensegs == 0) return; - reln->md_fd[forknum] = NULL; /* prevent dangling pointer after error */ - - while (v != NULL) + /* close segments starting from the end */ + while (nopensegs > 0) { - MdfdVec *ov = v; + MdfdVec *v = &reln->md_seg_fds[forknum][nopensegs - 1]; /* if not closed already */ if (v->mdfd_vfd >= 0) + { FileClose(v->mdfd_vfd); - /* Now free vector */ - v = v->mdfd_chain; - pfree(ov); + v->mdfd_vfd = -1; + } + + nopensegs--; } + + /* resize just once, avoids pointless reallocations */ + _fdvec_resize(reln, forknum, 0); } /* @@ -862,9 +864,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, * mdnblocks() -- Get the number of blocks stored in a relation. * * Important side effect: all active segments of the relation are opened - * and added to the mdfd_chain list. If this routine has not been + * and added to the mdfd_seg_fds array. If this routine has not been * called, then only segments up to the last one actually touched - * are present in the chain. + * are present in the array. */ BlockNumber mdnblocks(SMgrRelation reln, ForkNumber forknum) @@ -873,24 +875,24 @@ mdnblocks(SMgrRelation reln, ForkNumber forknum) BlockNumber nblocks; BlockNumber segno = 0; + /* mdopen has opened the first segment */ + Assert(reln->md_num_open_segs[forknum] > 0); + /* - * Skip through any segments that aren't the last one, to avoid redundant - * seeks on them. We have previously verified that these segments are - * exactly RELSEG_SIZE long, and it's useless to recheck that each time. + * Start from the last open segments, to avoid redundant seeks. We have + * previously verified that these segments are exactly RELSEG_SIZE long, + * and it's useless to recheck that each time. * * NOTE: this assumption could only be wrong if another backend has * truncated the relation. We rely on higher code levels to handle that * scenario by closing and re-opening the md fd, which is handled via * relcache flush. (Since the checkpointer doesn't participate in - * relcache flush, it could have segment chain entries for inactive - * segments; that's OK because the checkpointer never needs to compute - * relation size.) + * relcache flush, it could have segment entries for inactive segments; + * that's OK because the checkpointer never needs to compute relation + * size.) */ - while (v->mdfd_chain != NULL) - { - segno++; - v = v->mdfd_chain; - } + segno = reln->md_num_open_segs[forknum] - 1; + v = &reln->md_seg_fds[forknum][segno]; for (;;) { @@ -905,21 +907,16 @@ mdnblocks(SMgrRelation reln, ForkNumber forknum) */ segno++; - if (v->mdfd_chain == NULL) - { - /* - * We used to pass O_CREAT here, but that's has the disadvantage - * that it might create a segment which has vanished through some - * operating system misadventure. In such a case, creating the - * segment here undermines _mdfd_getseg's attempts to notice and - * report an error upon access to a missing segment. - */ - v->mdfd_chain = _mdfd_openseg(reln, forknum, segno, 0); - if (v->mdfd_chain == NULL) - return segno * ((BlockNumber) RELSEG_SIZE); - } - - v = v->mdfd_chain; + /* + * We used to pass O_CREAT here, but that's has the disadvantage that + * it might create a segment which has vanished through some operating + * system misadventure. In such a case, creating the segment here + * undermines _mdfd_getseg's attempts to notice and report an error + * upon access to a missing segment. + */ + v = _mdfd_openseg(reln, forknum, segno, 0); + if (v == NULL) + return segno * ((BlockNumber) RELSEG_SIZE); } } @@ -929,9 +926,9 @@ mdnblocks(SMgrRelation reln, ForkNumber forknum) void mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) { - MdfdVec *v; BlockNumber curnblk; BlockNumber priorblocks; + int curopensegs; /* * NOTE: mdnblocks makes sure we have opened all active segments, so that @@ -951,19 +948,24 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) if (nblocks == curnblk) return; /* no work */ - v = mdopen(reln, forknum, EXTENSION_FAIL); - - priorblocks = 0; - while (v != NULL) + /* + * Truncate segments, starting at the last one. Starting at the end makes + * managing the memory for the fd array easier, should there be errors. + */ + curopensegs = reln->md_num_open_segs[forknum]; + while (curopensegs > 0) { - MdfdVec *ov = v; + MdfdVec *v; + + priorblocks = (curopensegs - 1) * RELSEG_SIZE; + + v = &reln->md_seg_fds[forknum][curopensegs - 1]; if (priorblocks > nblocks) { /* - * This segment is no longer active (and has already been unlinked - * from the mdfd_chain). We truncate the file, but do not delete - * it, for reasons explained in the header comments. + * This segment is no longer active. We truncate the file, but do + * not delete it, for reasons explained in the header comments. */ if (FileTruncate(v->mdfd_vfd, 0) < 0) ereport(ERROR, @@ -973,21 +975,21 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) if (!SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); - v = v->mdfd_chain; - Assert(ov != reln->md_fd[forknum]); /* we never drop the 1st - * segment */ - FileClose(ov->mdfd_vfd); - pfree(ov); + + /* we never drop the 1st segment */ + Assert(v != &reln->md_seg_fds[forknum][0]); + + FileClose(v->mdfd_vfd); + _fdvec_resize(reln, forknum, curopensegs - 1); } else if (priorblocks + ((BlockNumber) RELSEG_SIZE) > nblocks) { /* * This is the last segment we want to keep. Truncate the file to - * the right length, and clear chain link that points to any - * remaining segments (which we shall zap). NOTE: if nblocks is - * exactly a multiple K of RELSEG_SIZE, we will truncate the K+1st - * segment to 0 length but keep it. This adheres to the invariant - * given in the header comments. + * the right length. NOTE: if nblocks is exactly a multiple K of + * RELSEG_SIZE, we will truncate the K+1st segment to 0 length but + * keep it. This adheres to the invariant given in the header + * comments. */ BlockNumber lastsegblocks = nblocks - priorblocks; @@ -999,18 +1001,16 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) nblocks))); if (!SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); - v = v->mdfd_chain; - ov->mdfd_chain = NULL; } else { /* - * We still need this segment and 0 or more blocks beyond it, so - * nothing to do here. + * We still need this segment, so nothing to do for this and any + * earlier segment. */ - v = v->mdfd_chain; + break; } - priorblocks += RELSEG_SIZE; + curopensegs--; } } @@ -1023,7 +1023,7 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) void mdimmedsync(SMgrRelation reln, ForkNumber forknum) { - MdfdVec *v; + int segno; /* * NOTE: mdnblocks makes sure we have opened all active segments, so that @@ -1031,16 +1031,18 @@ mdimmedsync(SMgrRelation reln, ForkNumber forknum) */ mdnblocks(reln, forknum); - v = mdopen(reln, forknum, EXTENSION_FAIL); + segno = reln->md_num_open_segs[forknum]; - while (v != NULL) + while (segno > 0) { + MdfdVec *v = &reln->md_seg_fds[forknum][segno - 1]; + if (FileSync(v->mdfd_vfd) < 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", FilePathName(v->mdfd_vfd)))); - v = v->mdfd_chain; + segno--; } } @@ -1703,12 +1705,40 @@ ForgetDatabaseFsyncRequests(Oid dbid) /* - * _fdvec_alloc() -- Make a MdfdVec object. + * _fdvec_resize() -- Resize the fork's open segments array */ -static MdfdVec * -_fdvec_alloc(void) +static void +_fdvec_resize(SMgrRelation reln, + ForkNumber forknum, + int nseg) { - return (MdfdVec *) MemoryContextAlloc(MdCxt, sizeof(MdfdVec)); + if (nseg == 0) + { + if (reln->md_num_open_segs[forknum] > 0) + { + pfree(reln->md_seg_fds[forknum]); + reln->md_seg_fds[forknum] = NULL; + } + } + else if (reln->md_num_open_segs[forknum] == 0) + { + reln->md_seg_fds[forknum] = + MemoryContextAlloc(MdCxt, sizeof(MdfdVec) * nseg); + } + else + { + /* + * It doesn't seem worthwile complicating the code by having a more + * aggressive growth strategy here; the number of segments doesn't + * grow that fast, and the memory context internally will sometimes + * avoid doing an actual reallocation. + */ + reln->md_seg_fds[forknum] = + repalloc(reln->md_seg_fds[forknum], + sizeof(MdfdVec) * nseg); + } + + reln->md_num_open_segs[forknum] = nseg; } /* @@ -1756,13 +1786,14 @@ _mdfd_openseg(SMgrRelation reln, ForkNumber forknum, BlockNumber segno, if (fd < 0) return NULL; - /* allocate an mdfdvec entry for it */ - v = _fdvec_alloc(); + if (segno <= reln->md_num_open_segs[forknum]) + _fdvec_resize(reln, forknum, segno + 1); /* fill the entry */ + v = &reln->md_seg_fds[forknum][segno]; v->mdfd_vfd = fd; v->mdfd_segno = segno; - v->mdfd_chain = NULL; + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); /* all done */ @@ -1781,7 +1812,7 @@ static MdfdVec * _mdfd_getseg(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, bool skipFsync, int behavior) { - MdfdVec *v = mdopen(reln, forknum, behavior); + MdfdVec *v; BlockNumber targetseg; BlockNumber nextsegno; @@ -1789,98 +1820,116 @@ _mdfd_getseg(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, Assert(behavior & (EXTENSION_FAIL | EXTENSION_CREATE | EXTENSION_RETURN_NULL)); - if (!v) - return NULL; /* if behavior & EXTENSION_RETURN_NULL */ - targetseg = blkno / ((BlockNumber) RELSEG_SIZE); - for (nextsegno = 1; nextsegno <= targetseg; nextsegno++) + + /* if an existing and opened segment, we're done */ + if (targetseg < reln->md_num_open_segs[forknum]) { - Assert(nextsegno == v->mdfd_segno + 1); + v = &reln->md_seg_fds[forknum][targetseg]; + return v; + } - if (v->mdfd_chain == NULL) - { - BlockNumber nblocks = _mdnblocks(reln, forknum, v); - int flags = 0; + /* + * The target segment is not yet open. Iterate over all the segments + * between the last opened and the target segment. This way missing + * segments either raise an error, or get created (according to + * 'behavior'). Start with either the last opened, or the first segment if + * none was opened before. + */ + if (reln->md_num_open_segs[forknum] > 0) + v = &reln->md_seg_fds[forknum][reln->md_num_open_segs[forknum] - 1]; + else + { + v = mdopen(reln, forknum, behavior); + if (!v) + return NULL; /* if behavior & EXTENSION_RETURN_NULL */ + } + + for (nextsegno = reln->md_num_open_segs[forknum]; + nextsegno <= targetseg; nextsegno++) + { + BlockNumber nblocks = _mdnblocks(reln, forknum, v); + int flags = 0; - if (nblocks > ((BlockNumber) RELSEG_SIZE)) - elog(FATAL, "segment too big"); + Assert(nextsegno == v->mdfd_segno + 1); + + if (nblocks > ((BlockNumber) RELSEG_SIZE)) + elog(FATAL, "segment too big"); - if ((behavior & EXTENSION_CREATE) || - (InRecovery && (behavior & EXTENSION_CREATE_RECOVERY))) + if ((behavior & EXTENSION_CREATE) || + (InRecovery && (behavior & EXTENSION_CREATE_RECOVERY))) + { + /* + * Normally we will create new segments only if authorized by the + * caller (i.e., we are doing mdextend()). But when doing WAL + * recovery, create segments anyway; this allows cases such as + * replaying WAL data that has a write into a high-numbered + * segment of a relation that was later deleted. We want to go + * ahead and create the segments so we can finish out the replay. + * However if the caller has specified + * EXTENSION_REALLY_RETURN_NULL, then extension is not desired + * even in recovery; we won't reach this point in that case. + * + * We have to maintain the invariant that segments before the last + * active segment are of size RELSEG_SIZE; therefore, if + * extending, pad them out with zeroes if needed. (This only + * matters if in recovery, or if the caller is extending the + * relation discontiguously, but that can happen in hash indexes.) + */ + if (nblocks < ((BlockNumber) RELSEG_SIZE)) { - /* - * Normally we will create new segments only if authorized by - * the caller (i.e., we are doing mdextend()). But when doing - * WAL recovery, create segments anyway; this allows cases - * such as replaying WAL data that has a write into a - * high-numbered segment of a relation that was later deleted. - * We want to go ahead and create the segments so we can - * finish out the replay. However if the caller has specified - * EXTENSION_REALLY_RETURN_NULL, then extension is not desired - * even in recovery; we won't reach this point in that case. - * - * We have to maintain the invariant that segments before the - * last active segment are of size RELSEG_SIZE; therefore, if - * extending, pad them out with zeroes if needed. (This only - * matters if in recovery, or if the caller is extending the - * relation discontiguously, but that can happen in hash - * indexes.) - */ - if (nblocks < ((BlockNumber) RELSEG_SIZE)) - { - char *zerobuf = palloc0(BLCKSZ); + char *zerobuf = palloc0(BLCKSZ); - mdextend(reln, forknum, - nextsegno * ((BlockNumber) RELSEG_SIZE) - 1, - zerobuf, skipFsync); - pfree(zerobuf); - } - flags = O_CREAT; + mdextend(reln, forknum, + nextsegno * ((BlockNumber) RELSEG_SIZE) - 1, + zerobuf, skipFsync); + pfree(zerobuf); } - else if (!(behavior & EXTENSION_DONT_CHECK_SIZE) && - nblocks < ((BlockNumber) RELSEG_SIZE)) + flags = O_CREAT; + } + else if (!(behavior & EXTENSION_DONT_CHECK_SIZE) && + nblocks < ((BlockNumber) RELSEG_SIZE)) + { + /* + * When not extending (or explicitly including truncated + * segments), only open the next segment if the current one is + * exactly RELSEG_SIZE. If not (this branch), either return NULL + * or fail. + */ + if (behavior & EXTENSION_RETURN_NULL) { /* - * When not extending (or explicitly including truncated - * segments), only open the next segment if the current one is - * exactly RELSEG_SIZE. If not (this branch), either return - * NULL or fail. + * Some callers discern between reasons for _mdfd_getseg() + * returning NULL based on errno. As there's no failing + * syscall involved in this case, explicitly set errno to + * ENOENT, as that seems the closest interpretation. */ - if (behavior & EXTENSION_RETURN_NULL) - { - /* - * Some callers discern between reasons for _mdfd_getseg() - * returning NULL based on errno. As there's no failing - * syscall involved in this case, explicitly set errno to - * ENOENT, as that seems the closest interpretation. - */ - errno = ENOENT; - return NULL; - } - - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open file \"%s\" (target block %u): previous segment is only %u blocks", - _mdfd_segpath(reln, forknum, nextsegno), - blkno, nblocks))); + errno = ENOENT; + return NULL; } - v->mdfd_chain = _mdfd_openseg(reln, forknum, nextsegno, flags); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" (target block %u): previous segment is only %u blocks", + _mdfd_segpath(reln, forknum, nextsegno), + blkno, nblocks))); + } - if (v->mdfd_chain == NULL) - { - if ((behavior & EXTENSION_RETURN_NULL) && - FILE_POSSIBLY_DELETED(errno)) - return NULL; - ereport(ERROR, - (errcode_for_file_access(), + v = _mdfd_openseg(reln, forknum, nextsegno, flags); + + if (v == NULL) + { + if ((behavior & EXTENSION_RETURN_NULL) && + FILE_POSSIBLY_DELETED(errno)) + return NULL; + ereport(ERROR, + (errcode_for_file_access(), errmsg("could not open file \"%s\" (target block %u): %m", _mdfd_segpath(reln, forknum, nextsegno), blkno))); - } } - v = v->mdfd_chain; } + return v; } diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c index 94aa952bcc..93f0080a60 100644 --- a/src/backend/storage/smgr/smgr.c +++ b/src/backend/storage/smgr/smgr.c @@ -174,7 +174,7 @@ smgropen(RelFileNode rnode, BackendId backend) /* mark it not open */ for (forknum = 0; forknum <= MAX_FORKNUM; forknum++) - reln->md_fd[forknum] = NULL; + reln->md_num_open_segs[forknum] = 0; /* it has no owner yet */ add_to_unowned_list(reln); @@ -379,7 +379,7 @@ smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo) * Exit quickly in WAL replay mode if we've already opened the file. If * it's open, it surely must exist. */ - if (isRedo && reln->md_fd[forknum] != NULL) + if (isRedo && reln->md_num_open_segs[forknum] > 0) return; /* diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h index a8e7877f70..3430d8665e 100644 --- a/src/include/storage/smgr.h +++ b/src/include/storage/smgr.h @@ -64,8 +64,12 @@ typedef struct SMgrRelationData */ int smgr_which; /* storage manager selector */ - /* for md.c; NULL for forks that are not open */ - struct _MdfdVec *md_fd[MAX_FORKNUM + 1]; + /* + * for md.c; per-fork arrays of the number of open segments + * (md_num_open_segs) and the segments themselves (md_seg_fds). + */ + int md_num_open_segs[MAX_FORKNUM + 1]; + struct _MdfdVec *md_seg_fds[MAX_FORKNUM + 1]; /* if unowned, list link in list of all unowned SMgrRelations */ struct SMgrRelationData *next_unowned_reln; From f66472428a51fc484bc5ca81791924d06a6f096d Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Fri, 9 Sep 2016 11:19:21 +0100 Subject: [PATCH 141/871] Correct TABLESAMPLE docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert to original use of word “sample”, though with clarification, per Tom Lane. Discussion: 29052.1471015383@sss.pgh.pa.us --- doc/src/sgml/ref/select.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/select.sgml b/doc/src/sgml/ref/select.sgml index e0098eb8d3..211e4c320c 100644 --- a/doc/src/sgml/ref/select.sgml +++ b/doc/src/sgml/ref/select.sgml @@ -391,7 +391,7 @@ TABLE [ ONLY ] table_name [ * ] not been changed meanwhile. But different seed values will usually produce different samples. If REPEATABLE is not given then a new random - seed is selected for each query. + sample is selected for each query, based upon a system-generated seed. Note that some add-on sampling methods do not accept REPEATABLE, and will always produce new samples on each use. From ec253de1fd2e6002122de80815ac5b963af8277c Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Fri, 9 Sep 2016 11:55:12 +0100 Subject: [PATCH 142/871] Fix corruption of 2PC recovery with subxacts Reading 2PC state files during recovery was borked, causing corruptions during recovery. Effect limited to servers with 2PC, subtransactions and recovery/replication. Stas Kelvich, reviewed by Michael Paquier and Pavan Deolasee --- src/backend/access/transam/twophase.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 1323fb508d..5415604993 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1758,8 +1758,9 @@ PrescanPreparedTransactions(TransactionId **xids_p, int *nxids_p) * need to hold a lock while examining it. We still acquire the * lock to modify it, though. */ - subxids = (TransactionId *) - (buf + MAXALIGN(sizeof(TwoPhaseFileHeader))); + subxids = (TransactionId *) (buf + + MAXALIGN(sizeof(TwoPhaseFileHeader)) + + MAXALIGN(hdr->gidlen)); for (i = 0; i < hdr->nsubxacts; i++) { TransactionId subxid = subxids[i]; @@ -1877,8 +1878,9 @@ StandbyRecoverPreparedTransactions(bool overwriteOK) * Examine subtransaction XIDs ... they should all follow main * XID. */ - subxids = (TransactionId *) - (buf + MAXALIGN(sizeof(TwoPhaseFileHeader))); + subxids = (TransactionId *) (buf + + MAXALIGN(sizeof(TwoPhaseFileHeader)) + + MAXALIGN(hdr->gidlen)); for (i = 0; i < hdr->nsubxacts; i++) { TransactionId subxid = subxids[i]; From 967a7b0fc9c8f4e07b697148238566203cb060de Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 9 Sep 2016 09:20:34 -0400 Subject: [PATCH 143/871] Avoid reporting "cache lookup failed" for some user-reachable cases. We have a not-terribly-thoroughly-enforced-yet project policy that internal errors with SQLSTATE XX000 (ie, plain elog) should not be triggerable from SQL. record_in, domain_in, and PL validator functions all failed to meet this standard, because they threw plain elog("cache lookup failed for XXX") errors on bad OIDs, and those are all invokable from SQL. For record_in, the best fix is to upgrade typcache.c (lookup_type_cache) to throw a user-facing error for this case. That seems consistent because it was more than halfway there already, having user-facing errors for shell types and non-composite types. Having done that, tweak domain_in to rely on the typcache to throw an appropriate error. (This costs little because InitDomainConstraintRef would fetch the typcache entry anyway.) For the PL validator functions, we already have a single choke point at CheckFunctionValidatorAccess, so just fix its error to be user-facing. Dilip Kumar, reviewed by Haribabu Kommi Discussion: <87wpxfygg9.fsf@credativ.de> --- src/backend/utils/adt/domains.c | 17 +++++++++++++---- src/backend/utils/cache/typcache.c | 16 +++++++++++----- src/backend/utils/fmgr/fmgr.c | 9 +++++++-- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/backend/utils/adt/domains.c b/src/backend/utils/adt/domains.c index 19ee4ce9d1..26bbbb5979 100644 --- a/src/backend/utils/adt/domains.c +++ b/src/backend/utils/adt/domains.c @@ -72,19 +72,28 @@ static DomainIOData * domain_state_setup(Oid domainType, bool binary, MemoryContext mcxt) { DomainIOData *my_extra; + TypeCacheEntry *typentry; Oid baseType; my_extra = (DomainIOData *) MemoryContextAlloc(mcxt, sizeof(DomainIOData)); - /* Find out the base type */ - my_extra->typtypmod = -1; - baseType = getBaseTypeAndTypmod(domainType, &my_extra->typtypmod); - if (baseType == domainType) + /* + * Verify that domainType represents a valid domain type. We need to be + * careful here because domain_in and domain_recv can be called from SQL, + * possibly with incorrect arguments. We use lookup_type_cache mainly + * because it will throw a clean user-facing error for a bad OID. + */ + typentry = lookup_type_cache(domainType, 0); + if (typentry->typtype != TYPTYPE_DOMAIN) ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("type %s is not a domain", format_type_be(domainType)))); + /* Find out the base type */ + my_extra->typtypmod = -1; + baseType = getBaseTypeAndTypmod(domainType, &my_extra->typtypmod); + /* Look up underlying I/O function */ if (binary) getTypeBinaryInputInfo(baseType, diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c index 9150fe832f..2d0db764e3 100644 --- a/src/backend/utils/cache/typcache.c +++ b/src/backend/utils/cache/typcache.c @@ -182,10 +182,10 @@ static int enum_oid_cmp(const void *left, const void *right); * Fetch the type cache entry for the specified datatype, and make sure that * all the fields requested by bits in 'flags' are valid. * - * The result is never NULL --- we will elog() if the passed type OID is + * The result is never NULL --- we will ereport() if the passed type OID is * invalid. Note however that we may fail to find one or more of the - * requested opclass-dependent fields; the caller needs to check whether - * the fields are InvalidOid or not. + * values requested by 'flags'; the caller needs to check whether the fields + * are InvalidOid or not. */ TypeCacheEntry * lookup_type_cache(Oid type_id, int flags) @@ -224,14 +224,18 @@ lookup_type_cache(Oid type_id, int flags) /* * If we didn't find one, we want to make one. But first look up the * pg_type row, just to make sure we don't make a cache entry for an - * invalid type OID. + * invalid type OID. If the type OID is not valid, present a + * user-facing error, since some code paths such as domain_in() allow + * this function to be reached with a user-supplied OID. */ HeapTuple tp; Form_pg_type typtup; tp = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type_id)); if (!HeapTupleIsValid(tp)) - elog(ERROR, "cache lookup failed for type %u", type_id); + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("type with OID %u does not exist", type_id))); typtup = (Form_pg_type) GETSTRUCT(tp); if (!typtup->typisdefined) ereport(ERROR, @@ -1230,6 +1234,8 @@ lookup_rowtype_tupdesc_internal(Oid type_id, int32 typmod, bool noError) * * Given a typeid/typmod that should describe a known composite type, * return the tuple descriptor for the type. Will ereport on failure. + * (Use ereport because this is reachable with user-specified OIDs, + * for example from record_in().) * * Note: on success, we increment the refcount of the returned TupleDesc, * and log the reference in CurrentResourceOwner. Caller should call diff --git a/src/backend/utils/fmgr/fmgr.c b/src/backend/utils/fmgr/fmgr.c index 7aae35074f..46a55ba7b9 100644 --- a/src/backend/utils/fmgr/fmgr.c +++ b/src/backend/utils/fmgr/fmgr.c @@ -2455,10 +2455,15 @@ CheckFunctionValidatorAccess(Oid validatorOid, Oid functionOid) Form_pg_language langStruct; AclResult aclresult; - /* Get the function's pg_proc entry */ + /* + * Get the function's pg_proc entry. Throw a user-facing error for bad + * OID, because validators can be called with user-specified OIDs. + */ procTup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); if (!HeapTupleIsValid(procTup)) - elog(ERROR, "cache lookup failed for function %u", functionOid); + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("function with OID %u does not exist", functionOid))); procStruct = (Form_pg_proc) GETSTRUCT(procTup); /* From e0013deb5983303d945aacd56909ac4ce227fde1 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 18 Aug 2016 12:00:00 -0400 Subject: [PATCH 144/871] Make better use of existing enums in plpgsql plpgsql.h defines a number of enums, but most of the code passes them around as ints. Update structs and function prototypes to take enum types instead. This clarifies the struct definitions in plpgsql.h in particular. Reviewed-by: Pavel Stehule --- src/pl/plpgsql/src/pl_comp.c | 6 +- src/pl/plpgsql/src/pl_exec.c | 2 +- src/pl/plpgsql/src/pl_funcs.c | 12 ++-- src/pl/plpgsql/src/plpgsql.h | 114 +++++++++++++++++----------------- 4 files changed, 67 insertions(+), 67 deletions(-) diff --git a/src/pl/plpgsql/src/pl_comp.c b/src/pl/plpgsql/src/pl_comp.c index 4ceb402c92..18948c8595 100644 --- a/src/pl/plpgsql/src/pl_comp.c +++ b/src/pl/plpgsql/src/pl_comp.c @@ -93,7 +93,7 @@ static PLpgSQL_function *do_compile(FunctionCallInfo fcinfo, PLpgSQL_func_hashkey *hashkey, bool forValidator); static void plpgsql_compile_error_callback(void *arg); -static void add_parameter_name(int itemtype, int itemno, const char *name); +static void add_parameter_name(PLpgSQL_nsitem_type itemtype, int itemno, const char *name); static void add_dummy_return(PLpgSQL_function *function); static Node *plpgsql_pre_column_ref(ParseState *pstate, ColumnRef *cref); static Node *plpgsql_post_column_ref(ParseState *pstate, ColumnRef *cref, Node *var); @@ -410,7 +410,7 @@ do_compile(FunctionCallInfo fcinfo, char argmode = argmodes ? argmodes[i] : PROARGMODE_IN; PLpgSQL_type *argdtype; PLpgSQL_variable *argvariable; - int argitemtype; + PLpgSQL_nsitem_type argitemtype; /* Create $n name for variable */ snprintf(buf, sizeof(buf), "$%d", i + 1); @@ -946,7 +946,7 @@ plpgsql_compile_error_callback(void *arg) * Add a name for a function parameter to the function's namespace */ static void -add_parameter_name(int itemtype, int itemno, const char *name) +add_parameter_name(PLpgSQL_nsitem_type itemtype, int itemno, const char *name) { /* * Before adding the name, check for duplicates. We need this even though diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index 2f8b6ff2f2..6141b7ab49 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -1557,7 +1557,7 @@ exec_stmt(PLpgSQL_execstate *estate, PLpgSQL_stmt *stmt) CHECK_FOR_INTERRUPTS(); - switch ((enum PLpgSQL_stmt_types) stmt->cmd_type) + switch (stmt->cmd_type) { case PLPGSQL_STMT_BLOCK: rc = exec_stmt_block(estate, (PLpgSQL_stmt_block *) stmt); diff --git a/src/pl/plpgsql/src/pl_funcs.c b/src/pl/plpgsql/src/pl_funcs.c index 27ebebce1e..e3cd9c0fc7 100644 --- a/src/pl/plpgsql/src/pl_funcs.c +++ b/src/pl/plpgsql/src/pl_funcs.c @@ -51,7 +51,7 @@ plpgsql_ns_init(void) * ---------- */ void -plpgsql_ns_push(const char *label, enum PLpgSQL_label_types label_type) +plpgsql_ns_push(const char *label, PLpgSQL_label_type label_type) { if (label == NULL) label = ""; @@ -89,7 +89,7 @@ plpgsql_ns_top(void) * ---------- */ void -plpgsql_ns_additem(int itemtype, int itemno, const char *name) +plpgsql_ns_additem(PLpgSQL_nsitem_type itemtype, int itemno, const char *name) { PLpgSQL_nsitem *nse; @@ -231,7 +231,7 @@ plpgsql_ns_find_nearest_loop(PLpgSQL_nsitem *ns_cur) const char * plpgsql_stmt_typename(PLpgSQL_stmt *stmt) { - switch ((enum PLpgSQL_stmt_types) stmt->cmd_type) + switch (stmt->cmd_type) { case PLPGSQL_STMT_BLOCK: return _("statement block"); @@ -291,7 +291,7 @@ plpgsql_stmt_typename(PLpgSQL_stmt *stmt) * GET DIAGNOSTICS item name as a string, for use in error messages etc. */ const char * -plpgsql_getdiag_kindname(int kind) +plpgsql_getdiag_kindname(PLpgSQL_getdiag_kind kind) { switch (kind) { @@ -367,7 +367,7 @@ static void free_expr(PLpgSQL_expr *expr); static void free_stmt(PLpgSQL_stmt *stmt) { - switch ((enum PLpgSQL_stmt_types) stmt->cmd_type) + switch (stmt->cmd_type) { case PLPGSQL_STMT_BLOCK: free_block((PLpgSQL_stmt_block *) stmt); @@ -791,7 +791,7 @@ static void dump_stmt(PLpgSQL_stmt *stmt) { printf("%3d:", stmt->lineno); - switch ((enum PLpgSQL_stmt_types) stmt->cmd_type) + switch (stmt->cmd_type) { case PLPGSQL_STMT_BLOCK: dump_block((PLpgSQL_stmt_block *) stmt); diff --git a/src/pl/plpgsql/src/plpgsql.h b/src/pl/plpgsql/src/plpgsql.h index b416e50c64..c84a97b675 100644 --- a/src/pl/plpgsql/src/plpgsql.h +++ b/src/pl/plpgsql/src/plpgsql.h @@ -37,28 +37,28 @@ /* * Compiler's namespace item types */ -enum +typedef enum PLpgSQL_nsitem_type { PLPGSQL_NSTYPE_LABEL, PLPGSQL_NSTYPE_VAR, PLPGSQL_NSTYPE_ROW, PLPGSQL_NSTYPE_REC -}; +} PLpgSQL_nsitem_type; /* * A PLPGSQL_NSTYPE_LABEL stack entry must be one of these types */ -enum PLpgSQL_label_types +typedef enum PLpgSQL_label_type { PLPGSQL_LABEL_BLOCK, /* DECLARE/BEGIN block */ PLPGSQL_LABEL_LOOP, /* looping construct */ PLPGSQL_LABEL_OTHER /* anything else */ -}; +} PLpgSQL_label_type; /* * Datum array node types */ -enum +typedef enum PLpgSQL_datum_type { PLPGSQL_DTYPE_VAR, PLPGSQL_DTYPE_ROW, @@ -66,23 +66,23 @@ enum PLPGSQL_DTYPE_RECFIELD, PLPGSQL_DTYPE_ARRAYELEM, PLPGSQL_DTYPE_EXPR -}; +} PLpgSQL_datum_type; /* * Variants distinguished in PLpgSQL_type structs */ -enum +typedef enum PLpgSQL_type_type { PLPGSQL_TTYPE_SCALAR, /* scalar types and domains */ PLPGSQL_TTYPE_ROW, /* composite types */ PLPGSQL_TTYPE_REC, /* RECORD pseudotype */ PLPGSQL_TTYPE_PSEUDO /* other pseudotypes */ -}; +} PLpgSQL_type_type; /* * Execution tree node types */ -enum PLpgSQL_stmt_types +typedef enum PLpgSQL_stmt_type { PLPGSQL_STMT_BLOCK, PLPGSQL_STMT_ASSIGN, @@ -108,7 +108,7 @@ enum PLpgSQL_stmt_types PLPGSQL_STMT_FETCH, PLPGSQL_STMT_CLOSE, PLPGSQL_STMT_PERFORM -}; +} PLpgSQL_stmt_type; /* * Execution node return codes @@ -124,7 +124,7 @@ enum /* * GET DIAGNOSTICS information items */ -enum +typedef enum PLpgSQL_getdiag_kind { PLPGSQL_GETDIAG_ROW_COUNT, PLPGSQL_GETDIAG_RESULT_OID, @@ -139,12 +139,12 @@ enum PLPGSQL_GETDIAG_MESSAGE_TEXT, PLPGSQL_GETDIAG_TABLE_NAME, PLPGSQL_GETDIAG_SCHEMA_NAME -}; +} PLpgSQL_getdiag_kind; /* * RAISE statement options */ -enum +typedef enum PLpgSQL_raise_option_type { PLPGSQL_RAISEOPTION_ERRCODE, PLPGSQL_RAISEOPTION_MESSAGE, @@ -155,12 +155,12 @@ enum PLPGSQL_RAISEOPTION_DATATYPE, PLPGSQL_RAISEOPTION_TABLE, PLPGSQL_RAISEOPTION_SCHEMA -}; +} PLpgSQL_raise_option_type; /* * Behavioral modes for plpgsql variable resolution */ -typedef enum +typedef enum PLpgSQL_resolve_option { PLPGSQL_RESOLVE_ERROR, /* throw error if ambiguous */ PLPGSQL_RESOLVE_VARIABLE, /* prefer plpgsql var to table column */ @@ -179,7 +179,7 @@ typedef struct PLpgSQL_type { char *typname; /* (simple) name of the type */ Oid typoid; /* OID of the data type */ - int ttype; /* PLPGSQL_TTYPE_ code */ + PLpgSQL_type_type ttype; /* PLPGSQL_TTYPE_ code */ int16 typlen; /* stuff copied from its pg_type entry */ bool typbyval; char typtype; @@ -197,7 +197,7 @@ typedef struct PLpgSQL_type */ typedef struct PLpgSQL_datum { - int dtype; + PLpgSQL_datum_type dtype; int dno; } PLpgSQL_datum; @@ -209,7 +209,7 @@ typedef struct PLpgSQL_datum */ typedef struct PLpgSQL_variable { - int dtype; + PLpgSQL_datum_type dtype; int dno; char *refname; int lineno; @@ -220,7 +220,7 @@ typedef struct PLpgSQL_variable */ typedef struct PLpgSQL_expr { - int dtype; + PLpgSQL_datum_type dtype; int dno; char *query; SPIPlanPtr plan; @@ -255,7 +255,7 @@ typedef struct PLpgSQL_expr */ typedef struct PLpgSQL_var { - int dtype; + PLpgSQL_datum_type dtype; int dno; char *refname; int lineno; @@ -278,7 +278,7 @@ typedef struct PLpgSQL_var */ typedef struct PLpgSQL_row { - int dtype; + PLpgSQL_datum_type dtype; int dno; char *refname; int lineno; @@ -301,7 +301,7 @@ typedef struct PLpgSQL_row */ typedef struct PLpgSQL_rec { - int dtype; + PLpgSQL_datum_type dtype; int dno; char *refname; int lineno; @@ -317,7 +317,7 @@ typedef struct PLpgSQL_rec */ typedef struct PLpgSQL_recfield { - int dtype; + PLpgSQL_datum_type dtype; int dno; char *fieldname; int recparentno; /* dno of parent record */ @@ -328,7 +328,7 @@ typedef struct PLpgSQL_recfield */ typedef struct PLpgSQL_arrayelem { - int dtype; + PLpgSQL_datum_type dtype; int dno; PLpgSQL_expr *subscript; int arrayparentno; /* dno of parent array variable */ @@ -350,9 +350,9 @@ typedef struct PLpgSQL_arrayelem */ typedef struct PLpgSQL_nsitem { - int itemtype; + PLpgSQL_nsitem_type itemtype; /* - * For labels, itemno is a value of enum PLpgSQL_label_types. For other + * For labels, itemno is a value of enum PLpgSQL_label_type. For other * itemtypes, itemno is the associated PLpgSQL_datum's dno. */ int itemno; @@ -365,7 +365,7 @@ typedef struct PLpgSQL_nsitem */ typedef struct PLpgSQL_stmt { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; } PLpgSQL_stmt; @@ -404,7 +404,7 @@ typedef struct PLpgSQL_exception */ typedef struct PLpgSQL_stmt_block { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; char *label; List *body; /* List of statements */ @@ -418,7 +418,7 @@ typedef struct PLpgSQL_stmt_block */ typedef struct PLpgSQL_stmt_assign { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; int varno; PLpgSQL_expr *expr; @@ -429,7 +429,7 @@ typedef struct PLpgSQL_stmt_assign */ typedef struct PLpgSQL_stmt_perform { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_expr *expr; } PLpgSQL_stmt_perform; @@ -439,7 +439,7 @@ typedef struct PLpgSQL_stmt_perform */ typedef struct PLpgSQL_diag_item { - int kind; /* id for diagnostic value desired */ + PLpgSQL_getdiag_kind kind; /* id for diagnostic value desired */ int target; /* where to assign it */ } PLpgSQL_diag_item; @@ -448,7 +448,7 @@ typedef struct PLpgSQL_diag_item */ typedef struct PLpgSQL_stmt_getdiag { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; bool is_stacked; /* STACKED or CURRENT diagnostics area? */ List *diag_items; /* List of PLpgSQL_diag_item */ @@ -459,7 +459,7 @@ typedef struct PLpgSQL_stmt_getdiag */ typedef struct PLpgSQL_stmt_if { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_expr *cond; /* boolean expression for THEN */ List *then_body; /* List of statements */ @@ -482,7 +482,7 @@ typedef struct PLpgSQL_if_elsif */ typedef struct PLpgSQL_stmt_case { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_expr *t_expr; /* test expression, or NULL if none */ int t_varno; /* var to store test expression value into */ @@ -506,7 +506,7 @@ typedef struct PLpgSQL_case_when */ typedef struct PLpgSQL_stmt_loop { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; char *label; List *body; /* List of statements */ @@ -517,7 +517,7 @@ typedef struct PLpgSQL_stmt_loop */ typedef struct PLpgSQL_stmt_while { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; char *label; PLpgSQL_expr *cond; @@ -529,7 +529,7 @@ typedef struct PLpgSQL_stmt_while */ typedef struct PLpgSQL_stmt_fori { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; char *label; PLpgSQL_var *var; @@ -547,7 +547,7 @@ typedef struct PLpgSQL_stmt_fori */ typedef struct PLpgSQL_stmt_forq { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; char *label; PLpgSQL_rec *rec; @@ -560,7 +560,7 @@ typedef struct PLpgSQL_stmt_forq */ typedef struct PLpgSQL_stmt_fors { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; char *label; PLpgSQL_rec *rec; @@ -575,7 +575,7 @@ typedef struct PLpgSQL_stmt_fors */ typedef struct PLpgSQL_stmt_forc { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; char *label; PLpgSQL_rec *rec; @@ -591,7 +591,7 @@ typedef struct PLpgSQL_stmt_forc */ typedef struct PLpgSQL_stmt_dynfors { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; char *label; PLpgSQL_rec *rec; @@ -607,7 +607,7 @@ typedef struct PLpgSQL_stmt_dynfors */ typedef struct PLpgSQL_stmt_foreach_a { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; char *label; int varno; /* loop target variable */ @@ -621,7 +621,7 @@ typedef struct PLpgSQL_stmt_foreach_a */ typedef struct PLpgSQL_stmt_open { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; int curvar; int cursor_options; @@ -637,7 +637,7 @@ typedef struct PLpgSQL_stmt_open */ typedef struct PLpgSQL_stmt_fetch { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_rec *rec; /* target, as record or row */ PLpgSQL_row *row; @@ -654,7 +654,7 @@ typedef struct PLpgSQL_stmt_fetch */ typedef struct PLpgSQL_stmt_close { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; int curvar; } PLpgSQL_stmt_close; @@ -664,7 +664,7 @@ typedef struct PLpgSQL_stmt_close */ typedef struct PLpgSQL_stmt_exit { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; bool is_exit; /* Is this an exit or a continue? */ char *label; /* NULL if it's an unlabelled EXIT/CONTINUE */ @@ -676,7 +676,7 @@ typedef struct PLpgSQL_stmt_exit */ typedef struct PLpgSQL_stmt_return { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_expr *expr; int retvarno; @@ -687,7 +687,7 @@ typedef struct PLpgSQL_stmt_return */ typedef struct PLpgSQL_stmt_return_next { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_expr *expr; int retvarno; @@ -698,7 +698,7 @@ typedef struct PLpgSQL_stmt_return_next */ typedef struct PLpgSQL_stmt_return_query { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_expr *query; /* if static query */ PLpgSQL_expr *dynquery; /* if dynamic query (RETURN QUERY EXECUTE) */ @@ -710,7 +710,7 @@ typedef struct PLpgSQL_stmt_return_query */ typedef struct PLpgSQL_stmt_raise { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; int elog_level; char *condname; /* condition name, SQLSTATE, or NULL */ @@ -724,7 +724,7 @@ typedef struct PLpgSQL_stmt_raise */ typedef struct PLpgSQL_raise_option { - int opt_type; + PLpgSQL_raise_option_type opt_type; PLpgSQL_expr *expr; } PLpgSQL_raise_option; @@ -733,7 +733,7 @@ typedef struct PLpgSQL_raise_option */ typedef struct PLpgSQL_stmt_assert { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_expr *cond; PLpgSQL_expr *message; @@ -744,7 +744,7 @@ typedef struct PLpgSQL_stmt_assert */ typedef struct PLpgSQL_stmt_execsql { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_expr *sqlstmt; bool mod_stmt; /* is the stmt INSERT/UPDATE/DELETE? Note: @@ -760,7 +760,7 @@ typedef struct PLpgSQL_stmt_execsql */ typedef struct PLpgSQL_stmt_dynexecute { - int cmd_type; + PLpgSQL_stmt_type cmd_type; int lineno; PLpgSQL_expr *query; /* string expression */ bool into; /* INTO supplied? */ @@ -1111,10 +1111,10 @@ extern void plpgsql_exec_get_datum_type_info(PLpgSQL_execstate *estate, */ extern void plpgsql_ns_init(void); extern void plpgsql_ns_push(const char *label, - enum PLpgSQL_label_types label_type); + PLpgSQL_label_type label_type); extern void plpgsql_ns_pop(void); extern PLpgSQL_nsitem *plpgsql_ns_top(void); -extern void plpgsql_ns_additem(int itemtype, int itemno, const char *name); +extern void plpgsql_ns_additem(PLpgSQL_nsitem_type itemtype, int itemno, const char *name); extern PLpgSQL_nsitem *plpgsql_ns_lookup(PLpgSQL_nsitem *ns_cur, bool localmode, const char *name1, const char *name2, const char *name3, int *names_used); @@ -1126,7 +1126,7 @@ extern PLpgSQL_nsitem *plpgsql_ns_find_nearest_loop(PLpgSQL_nsitem *ns_cur); * Other functions in pl_funcs.c */ extern const char *plpgsql_stmt_typename(PLpgSQL_stmt *stmt); -extern const char *plpgsql_getdiag_kindname(int kind); +extern const char *plpgsql_getdiag_kindname(PLpgSQL_getdiag_kind kind); extern void plpgsql_free_function_memory(PLpgSQL_function *func); extern void plpgsql_dumptree(PLpgSQL_function *func); From 984d0a14e8d0141a68da5bd56ce6821042298904 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 9 Sep 2016 12:20:58 -0400 Subject: [PATCH 145/871] In PageIndexTupleDelete, don't assume stored item lengths are MAXALIGNed. PageAddItem stores the item length as-is. It MAXALIGN's the amount of space actually allocated for each tuple, but not the stored length. PageRepairFragmentation, PageIndexMultiDelete, and PageIndexDeleteNoCompact are all on board with this and MAXALIGN item lengths after fetching them. But PageIndexTupleDelete expects the stored length to be a MAXALIGN multiple already. This accidentally works for existing index AMs because they all maxalign their tuple sizes internally; but we don't do that for heap tuples, and it shouldn't be a requirement for index tuples either. So, sync PageIndexTupleDelete with the rest of bufpage.c by having it maxalign the item size after fetching. Also add a check that pd_special is maxaligned, to ensure that the test "(offset + size) > phdr->pd_special" is still doing the right thing. (If offset and pd_special are aligned, it doesn't matter whether size is.) Again, this is in sync with the rest of the routines here, except for PageAddItem which doesn't test because it doesn't actually do anything for which pd_special alignment matters. This shouldn't have any immediate functional impact; it just adds the flexibility to use PageIndexTupleDelete on index tuples with non-aligned lengths. Discussion: <3814.1473366762@sss.pgh.pa.us> --- src/backend/storage/page/bufpage.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index 1b70bfbe8c..bce0d53a9c 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -738,7 +738,8 @@ PageIndexTupleDelete(Page page, OffsetNumber offnum) if (phdr->pd_lower < SizeOfPageHeaderData || phdr->pd_lower > phdr->pd_upper || phdr->pd_upper > phdr->pd_special || - phdr->pd_special > BLCKSZ) + phdr->pd_special > BLCKSZ || + phdr->pd_special != MAXALIGN(phdr->pd_special)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupted page pointers: lower = %u, upper = %u, special = %u", @@ -757,12 +758,15 @@ PageIndexTupleDelete(Page page, OffsetNumber offnum) offset = ItemIdGetOffset(tup); if (offset < phdr->pd_upper || (offset + size) > phdr->pd_special || - offset != MAXALIGN(offset) || size != MAXALIGN(size)) + offset != MAXALIGN(offset)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupted item pointer: offset = %u, size = %u", offset, (unsigned int) size))); + /* Amount of space to actually be deleted */ + size = MAXALIGN(size); + /* * First, we want to get rid of the pd_linp entry for the index tuple. We * copy all subsequent linp's back one slot in the array. We don't use From 5c609a742f294907512b946dbaf1feaa3b71ddc7 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 9 Sep 2016 15:54:29 -0300 Subject: [PATCH 146/871] Fix locking a tuple updated by an aborted (sub)transaction When heap_lock_tuple decides to follow the update chain, it tried to also lock any version of the tuple that was created by an update that was subsequently rolled back. This is pointless, since for all intents and purposes that tuple exists no more; and moreover it causes misbehavior, as reported independently by Marko Tiikkaja and Marti Raudsepp: some SELECT FOR UPDATE/SHARE queries may fail to return the tuples, and assertion-enabled builds crash. Fix by having heap_lock_updated_tuple test the xmin and return success immediately if the tuple was created by an aborted transaction. The condition where tuples become invisible occurs when an updated tuple chain is followed by heap_lock_updated_tuple, which reports the problem as HeapTupleSelfUpdated to its caller heap_lock_tuple, which in turn propagates that code outwards possibly leading the calling code (ExecLockRows) to believe that the tuple exists no longer. Backpatch to 9.3. Only on 9.5 and newer this leads to a visible failure, because of commit 27846f02c176; before that, heap_lock_tuple skips the whole dance when the tuple is already locked by the same transaction, because of the ancient HeapTupleSatisfiesUpdate behavior. Still, the buggy condition may also exist in more convoluted scenarios involving concurrent transactions, so it seems safer to fix the bug in the old branches too. Discussion: https://www.postgresql.org/message-id/CABRT9RC81YUf1=jsmWopcKJEro=VoeG2ou6sPwyOUTx_qteRsg@mail.gmail.com https://www.postgresql.org/message-id/48d3eade-98d3-8b9a-477e-1a8dc32a724d@joh.to --- src/backend/access/heap/heapam.c | 11 +++++++++++ src/test/regress/expected/combocid.out | 27 ++++++++++++++++++++++++++ src/test/regress/sql/combocid.sql | 18 +++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 6a27ef4140..b019bc1a0d 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -5722,6 +5722,17 @@ heap_lock_updated_tuple_rec(Relation rel, ItemPointer tid, TransactionId xid, goto out_locked; } + /* + * Also check Xmin: if this tuple was created by an aborted + * (sub)transaction, then we already locked the last live one in the + * chain, thus we're done, so return success. + */ + if (TransactionIdDidAbort(HeapTupleHeaderGetXmin(mytup.t_data))) + { + UnlockReleaseBuffer(buf); + return HeapTupleMayBeUpdated; + } + old_infomask = mytup.t_data->t_infomask; old_infomask2 = mytup.t_data->t_infomask2; xmax = HeapTupleHeaderGetRawXmax(mytup.t_data); diff --git a/src/test/regress/expected/combocid.out b/src/test/regress/expected/combocid.out index b63894c283..17eb94a8ff 100644 --- a/src/test/regress/expected/combocid.out +++ b/src/test/regress/expected/combocid.out @@ -140,3 +140,30 @@ SELECT ctid,cmin,* FROM combocidtest; (0,6) | 0 | 444 (3 rows) +-- test for bug reported in +-- CABRT9RC81YUf1=jsmWopcKJEro=VoeG2ou6sPwyOUTx_qteRsg@mail.gmail.com +CREATE TABLE IF NOT EXISTS testcase( + id int PRIMARY KEY, + balance numeric +); +INSERT INTO testcase VALUES (1, 0); +BEGIN; +SELECT * FROM testcase WHERE testcase.id = 1 FOR UPDATE; + id | balance +----+--------- + 1 | 0 +(1 row) + +UPDATE testcase SET balance = balance + 400 WHERE id=1; +SAVEPOINT subxact; +UPDATE testcase SET balance = balance - 100 WHERE id=1; +ROLLBACK TO SAVEPOINT subxact; +-- should return one tuple +SELECT * FROM testcase WHERE id = 1 FOR UPDATE; + id | balance +----+--------- + 1 | 400 +(1 row) + +ROLLBACK; +DROP TABLE testcase; diff --git a/src/test/regress/sql/combocid.sql b/src/test/regress/sql/combocid.sql index f24ac6b01a..4faea36f41 100644 --- a/src/test/regress/sql/combocid.sql +++ b/src/test/regress/sql/combocid.sql @@ -91,3 +91,21 @@ SELECT ctid,cmin,* FROM combocidtest; COMMIT; SELECT ctid,cmin,* FROM combocidtest; + +-- test for bug reported in +-- CABRT9RC81YUf1=jsmWopcKJEro=VoeG2ou6sPwyOUTx_qteRsg@mail.gmail.com +CREATE TABLE IF NOT EXISTS testcase( + id int PRIMARY KEY, + balance numeric +); +INSERT INTO testcase VALUES (1, 0); +BEGIN; +SELECT * FROM testcase WHERE testcase.id = 1 FOR UPDATE; +UPDATE testcase SET balance = balance + 400 WHERE id=1; +SAVEPOINT subxact; +UPDATE testcase SET balance = balance - 100 WHERE id=1; +ROLLBACK TO SAVEPOINT subxact; +-- should return one tuple +SELECT * FROM testcase WHERE id = 1 FOR UPDATE; +ROLLBACK; +DROP TABLE testcase; From b1328d78f88cdf4f7504004159e530b776f0de16 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 9 Sep 2016 18:02:24 -0400 Subject: [PATCH 147/871] Invent PageIndexTupleOverwrite, and teach BRIN and GiST to use it. PageIndexTupleOverwrite performs approximately the same function as PageIndexTupleDelete (or PageIndexDeleteNoCompact) followed by PageAddItem targeting the same item pointer offset. But in the case where the new tuple is the same size as the old, it avoids shuffling other data around on the page, because the new tuple is placed where the old one was rather than being appended to the end of the page. This has been shown to provide a substantial speedup for some GiST use-cases. Also, this change allows some API simplifications: we can get rid of the rather klugy and error-prone PAI_ALLOW_FAR_OFFSET flag for PageAddItemExtended, since that was used only to cover a corner case for BRIN that's better expressed by using PageIndexTupleOverwrite. Note that this patch causes a rather subtle WAL incompatibility: the physical page content change represented by certain WAL records is now different than it was before, because while the tuples have the same itempointer line numbers, the tuples themselves are in different places. I have not bumped the WAL version number because I think it doesn't matter unless you are trying to do bitwise comparisons of original and replayed pages, and in any case we're early in a devel cycle and there will probably be more WAL changes before v10 gets out the door. There is probably room to make use of PageIndexTupleOverwrite in SP-GiST and GIN too, but that is left for a future patch. Andrey Borodin, reviewed by Anastasia Lubennikova, whacked around a bit by me Discussion: --- src/backend/access/brin/brin_pageops.c | 6 +- src/backend/access/brin/brin_xlog.c | 9 +- src/backend/access/gist/gist.c | 30 ++++-- src/backend/access/gist/gistxlog.c | 29 ++++- src/backend/storage/page/bufpage.c | 142 +++++++++++++++++++++---- src/include/storage/bufpage.h | 3 +- 6 files changed, 179 insertions(+), 40 deletions(-) diff --git a/src/backend/access/brin/brin_pageops.c b/src/backend/access/brin/brin_pageops.c index 6ebfedd6a9..24ae38be65 100644 --- a/src/backend/access/brin/brin_pageops.c +++ b/src/backend/access/brin/brin_pageops.c @@ -178,10 +178,8 @@ brin_doupdate(Relation idxrel, BlockNumber pagesPerRange, } START_CRIT_SECTION(); - PageIndexDeleteNoCompact(oldpage, &oldoff, 1); - if (PageAddItemExtended(oldpage, (Item) newtup, newsz, oldoff, - PAI_OVERWRITE | PAI_ALLOW_FAR_OFFSET) == InvalidOffsetNumber) - elog(ERROR, "failed to add BRIN tuple"); + if (!PageIndexTupleOverwrite(oldpage, oldoff, (Item) newtup, newsz)) + elog(ERROR, "failed to replace BRIN tuple"); MarkBufferDirty(oldbuf); /* XLOG stuff */ diff --git a/src/backend/access/brin/brin_xlog.c b/src/backend/access/brin/brin_xlog.c index 27ba0a97f8..ed14729955 100644 --- a/src/backend/access/brin/brin_xlog.c +++ b/src/backend/access/brin/brin_xlog.c @@ -189,14 +189,9 @@ brin_xlog_samepage_update(XLogReaderState *record) page = (Page) BufferGetPage(buffer); offnum = xlrec->offnum; - if (PageGetMaxOffsetNumber(page) + 1 < offnum) - elog(PANIC, "brin_xlog_samepage_update: invalid max offset number"); - PageIndexDeleteNoCompact(page, &offnum, 1); - offnum = PageAddItemExtended(page, (Item) brintuple, tuplen, offnum, - PAI_OVERWRITE | PAI_ALLOW_FAR_OFFSET); - if (offnum == InvalidOffsetNumber) - elog(PANIC, "brin_xlog_samepage_update: failed to add tuple"); + if (!PageIndexTupleOverwrite(page, offnum, (Item) brintuple, tuplen)) + elog(PANIC, "brin_xlog_samepage_update: failed to replace tuple"); PageSetLSN(page, lsn); MarkBufferDirty(buffer); diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index f7f44b49aa..b8aa9bca60 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -493,18 +493,36 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, else { /* - * Enough space. We also get here if ntuples==0. + * Enough space. We always get here if ntup==0. */ START_CRIT_SECTION(); /* - * While we delete only one tuple at once we could mix calls - * PageIndexTupleDelete() here and PageIndexMultiDelete() in - * gistRedoPageUpdateRecord() + * Delete old tuple if any, then insert new tuple(s) if any. If + * possible, use the fast path of PageIndexTupleOverwrite. */ if (OffsetNumberIsValid(oldoffnum)) - PageIndexTupleDelete(page, oldoffnum); - gistfillbuffer(page, itup, ntup, InvalidOffsetNumber); + { + if (ntup == 1) + { + /* One-for-one replacement, so use PageIndexTupleOverwrite */ + if (!PageIndexTupleOverwrite(page, oldoffnum, (Item) *itup, + IndexTupleSize(*itup))) + elog(ERROR, "failed to add item to index page in \"%s\"", + RelationGetRelationName(rel)); + } + else + { + /* Delete old, then append new tuple(s) to page */ + PageIndexTupleDelete(page, oldoffnum); + gistfillbuffer(page, itup, ntup, InvalidOffsetNumber); + } + } + else + { + /* Just append new tuples at the end of the page */ + gistfillbuffer(page, itup, ntup, InvalidOffsetNumber); + } MarkBufferDirty(buffer); diff --git a/src/backend/access/gist/gistxlog.c b/src/backend/access/gist/gistxlog.c index 01c7ef7ea6..5853d7638e 100644 --- a/src/backend/access/gist/gistxlog.c +++ b/src/backend/access/gist/gistxlog.c @@ -80,9 +80,31 @@ gistRedoPageUpdateRecord(XLogReaderState *record) page = (Page) BufferGetPage(buffer); - /* Delete old tuples */ - if (xldata->ntodelete > 0) + if (xldata->ntodelete == 1 && xldata->ntoinsert == 1) { + /* + * When replacing one tuple with one other tuple, we must use + * PageIndexTupleOverwrite for consistency with gistplacetopage. + */ + OffsetNumber offnum = *((OffsetNumber *) data); + IndexTuple itup; + Size itupsize; + + data += sizeof(OffsetNumber); + itup = (IndexTuple) data; + itupsize = IndexTupleSize(itup); + if (!PageIndexTupleOverwrite(page, offnum, (Item) itup, itupsize)) + elog(ERROR, "failed to add item to GiST index page, size %d bytes", + (int) itupsize); + data += itupsize; + /* should be nothing left after consuming 1 tuple */ + Assert(data - begin == datalen); + /* update insertion count for assert check below */ + ninserted++; + } + else if (xldata->ntodelete > 0) + { + /* Otherwise, delete old tuples if any */ OffsetNumber *todelete = (OffsetNumber *) data; data += sizeof(OffsetNumber) * xldata->ntodelete; @@ -92,7 +114,7 @@ gistRedoPageUpdateRecord(XLogReaderState *record) GistMarkTuplesDeleted(page); } - /* add tuples */ + /* Add new tuples if any */ if (data - begin < datalen) { OffsetNumber off = (PageIsEmpty(page)) ? FirstOffsetNumber : @@ -115,6 +137,7 @@ gistRedoPageUpdateRecord(XLogReaderState *record) } } + /* Check that XLOG record contained expected number of tuples */ Assert(ninserted == xldata->ntoinsert); PageSetLSN(page, lsn); diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index bce0d53a9c..08e222e583 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -166,21 +166,24 @@ PageIsVerified(Page page, BlockNumber blkno) * inserted, or InvalidOffsetNumber if the item is not inserted for any * reason. A WARNING is issued indicating the reason for the refusal. * - * If flag PAI_OVERWRITE is set, we just store the item at the specified - * offsetNumber (which must be either a currently-unused item pointer, - * or one past the last existing item). Otherwise, - * if offsetNumber is valid and <= current max offset in the page, - * insert item into the array at that position by shuffling ItemId's - * down to make room. - * If offsetNumber is not valid, then assign one by finding the first + * offsetNumber must be either InvalidOffsetNumber to specify finding a + * free item pointer, or a value between FirstOffsetNumber and one past + * the last existing item, to specify using that particular item pointer. + * + * If offsetNumber is valid and flag PAI_OVERWRITE is set, we just store + * the item at the specified offsetNumber, which must be either a + * currently-unused item pointer, or one past the last existing item. + * + * If offsetNumber is valid and flag PAI_OVERWRITE is not set, insert + * the item at the specified offsetNumber, moving existing items later + * in the array to make room. + * + * If offsetNumber is not valid, then assign a slot by finding the first * one that is both unused and deallocated. * * If flag PAI_IS_HEAP is set, we enforce that there can't be more than * MaxHeapTuplesPerPage line pointers on the page. * - * If flag PAI_ALLOW_FAR_OFFSET is not set, we disallow placing items - * beyond one past the last existing item. - * * !!! EREPORT(ERROR) IS DISALLOWED HERE !!! */ OffsetNumber @@ -267,11 +270,8 @@ PageAddItemExtended(Page page, } } - /* - * Reject placing items beyond the first unused line pointer, unless - * caller asked for that behavior specifically. - */ - if ((flags & PAI_ALLOW_FAR_OFFSET) == 0 && offsetNumber > limit) + /* Reject placing items beyond the first unused line pointer */ + if (offsetNumber > limit) { elog(WARNING, "specified item offset is too large"); return InvalidOffsetNumber; @@ -290,10 +290,7 @@ PageAddItemExtended(Page page, * Note: do arithmetic as signed ints, to avoid mistakes if, say, * alignedSize > pd_upper. */ - if ((flags & PAI_ALLOW_FAR_OFFSET) != 0) - lower = Max(phdr->pd_lower, - SizeOfPageHeaderData + sizeof(ItemIdData) * offsetNumber); - else if (offsetNumber == limit || needshuffle) + if (offsetNumber == limit || needshuffle) lower = phdr->pd_lower + sizeof(ItemIdData); else lower = phdr->pd_lower; @@ -1093,6 +1090,113 @@ PageIndexDeleteNoCompact(Page page, OffsetNumber *itemnos, int nitems) } } + +/* + * PageIndexTupleOverwrite + * + * Replace a specified tuple on an index page. + * + * The new tuple is placed exactly where the old one had been, shifting + * other tuples' data up or down as needed to keep the page compacted. + * This is better than deleting and reinserting the tuple, because it + * avoids any data shifting when the tuple size doesn't change; and + * even when it does, we avoid moving the item pointers around. + * Conceivably this could also be of use to an index AM that cares about + * the physical order of tuples as well as their ItemId order. + * + * If there's insufficient space for the new tuple, return false. Other + * errors represent data-corruption problems, so we just elog. + */ +bool +PageIndexTupleOverwrite(Page page, OffsetNumber offnum, + Item newtup, Size newsize) +{ + PageHeader phdr = (PageHeader) page; + ItemId tupid; + int oldsize; + unsigned offset; + Size alignednewsize; + int size_diff; + int itemcount; + + /* + * As with PageRepairFragmentation, paranoia seems justified. + */ + if (phdr->pd_lower < SizeOfPageHeaderData || + phdr->pd_lower > phdr->pd_upper || + phdr->pd_upper > phdr->pd_special || + phdr->pd_special > BLCKSZ || + phdr->pd_special != MAXALIGN(phdr->pd_special)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("corrupted page pointers: lower = %u, upper = %u, special = %u", + phdr->pd_lower, phdr->pd_upper, phdr->pd_special))); + + itemcount = PageGetMaxOffsetNumber(page); + if ((int) offnum <= 0 || (int) offnum > itemcount) + elog(ERROR, "invalid index offnum: %u", offnum); + + tupid = PageGetItemId(page, offnum); + Assert(ItemIdHasStorage(tupid)); + oldsize = ItemIdGetLength(tupid); + offset = ItemIdGetOffset(tupid); + + if (offset < phdr->pd_upper || (offset + oldsize) > phdr->pd_special || + offset != MAXALIGN(offset)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("corrupted item pointer: offset = %u, size = %u", + offset, (unsigned int) oldsize))); + + /* + * Determine actual change in space requirement, check for page overflow. + */ + oldsize = MAXALIGN(oldsize); + alignednewsize = MAXALIGN(newsize); + if (alignednewsize > oldsize + (phdr->pd_upper - phdr->pd_lower)) + return false; + + /* + * Relocate existing data and update line pointers, unless the new tuple + * is the same size as the old (after alignment), in which case there's + * nothing to do. Notice that what we have to relocate is data before the + * target tuple, not data after, so it's convenient to express size_diff + * as the amount by which the tuple's size is decreasing, making it the + * delta to add to pd_upper and affected line pointers. + */ + size_diff = oldsize - (int) alignednewsize; + if (size_diff != 0) + { + char *addr = (char *) page + phdr->pd_upper; + int i; + + /* relocate all tuple data before the target tuple */ + memmove(addr + size_diff, addr, offset - phdr->pd_upper); + + /* adjust free space boundary pointer */ + phdr->pd_upper += size_diff; + + /* adjust affected line pointers too */ + for (i = FirstOffsetNumber; i <= itemcount; i++) + { + ItemId ii = PageGetItemId(phdr, i); + + /* Allow items without storage; currently only BRIN needs that */ + if (ItemIdHasStorage(ii) && ItemIdGetOffset(ii) <= offset) + ii->lp_off += size_diff; + } + } + + /* Update the item's tuple length (other fields shouldn't change) */ + ItemIdSetNormal(tupid, offset + size_diff, newsize); + + /* Copy new tuple data onto page */ + memcpy(PageGetItem(page, tupid), newtup, newsize); + + return true; +} + + /* * Set checksum for a page in shared buffers. * diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h index 15cebfc60d..0ea47f5457 100644 --- a/src/include/storage/bufpage.h +++ b/src/include/storage/bufpage.h @@ -409,7 +409,6 @@ do { \ */ #define PAI_OVERWRITE (1 << 0) #define PAI_IS_HEAP (1 << 1) -#define PAI_ALLOW_FAR_OFFSET (1 << 2) extern void PageInit(Page page, Size pageSize, Size specialSize); extern bool PageIsVerified(Page page, BlockNumber blkno); @@ -429,6 +428,8 @@ extern void PageIndexTupleDelete(Page page, OffsetNumber offset); extern void PageIndexMultiDelete(Page page, OffsetNumber *itemnos, int nitems); extern void PageIndexDeleteNoCompact(Page page, OffsetNumber *itemnos, int nitems); +extern bool PageIndexTupleOverwrite(Page page, OffsetNumber offnum, + Item newtup, Size newsize); extern char *PageSetChecksumCopy(Page page, BlockNumber blkno); extern void PageSetChecksumInplace(Page page, BlockNumber blkno); From 1a4be103a523db8d47b464463ba175cc664442b2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 9 Sep 2016 18:17:07 -0400 Subject: [PATCH 148/871] Convert PageAddItem into a macro to save a few cycles. Nowadays this is just a backwards-compatibility wrapper around PageAddItemExtended, so let's avoid the extra level of function call. In addition, because pretty much all callers are passing constants for the two bool arguments, compilers will be able to constant-fold the conversion to a flags bitmask. Discussion: <552.1473445163@sss.pgh.pa.us> --- src/backend/storage/page/bufpage.c | 20 -------------------- src/include/storage/bufpage.h | 7 +++++-- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index 08e222e583..41a7dd0d62 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -338,26 +338,6 @@ PageAddItemExtended(Page page, return offsetNumber; } -/* - * PageAddItem - * - * Add an item to a page. Return value is offset at which it was - * inserted, or InvalidOffsetNumber if the item is not inserted for - * any reason. - * - * Passing the 'overwrite' and 'is_heap' parameters as true causes the - * PAI_OVERWRITE and PAI_IS_HEAP flags to be set, respectively. - * - * !!! EREPORT(ERROR) IS DISALLOWED HERE !!! - */ -OffsetNumber -PageAddItem(Page page, Item item, Size size, OffsetNumber offsetNumber, - bool overwrite, bool is_heap) -{ - return PageAddItemExtended(page, item, size, offsetNumber, - overwrite ? PAI_OVERWRITE : 0 | - is_heap ? PAI_IS_HEAP : 0); -} /* * PageGetTempPage diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h index 0ea47f5457..4cedff7fa4 100644 --- a/src/include/storage/bufpage.h +++ b/src/include/storage/bufpage.h @@ -410,10 +410,13 @@ do { \ #define PAI_OVERWRITE (1 << 0) #define PAI_IS_HEAP (1 << 1) +#define PageAddItem(page, item, size, offsetNumber, overwrite, is_heap) \ + PageAddItemExtended(page, item, size, offsetNumber, \ + ((overwrite) ? PAI_OVERWRITE : 0) | \ + ((is_heap) ? PAI_IS_HEAP : 0)) + extern void PageInit(Page page, Size pageSize, Size specialSize); extern bool PageIsVerified(Page page, BlockNumber blkno); -extern OffsetNumber PageAddItem(Page page, Item item, Size size, - OffsetNumber offsetNumber, bool overwrite, bool is_heap); extern OffsetNumber PageAddItemExtended(Page page, Item item, Size size, OffsetNumber offsetNumber, int flags); extern Page PageGetTempPage(Page page); From 24992c6db9fd40f342db1f22747ec9e56483796d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 9 Sep 2016 19:00:59 -0400 Subject: [PATCH 149/871] Rewrite PageIndexDeleteNoCompact into a form that only deletes 1 tuple. The full generality of deleting an arbitrary number of tuples is no longer needed, so let's save some code and cycles by replacing the original coding with an implementation based on PageIndexTupleDelete. We can always get back the old code from git if we need it again for new callers (though I don't care for its willingness to mess with line pointers it wasn't told to mess with). Discussion: <552.1473445163@sss.pgh.pa.us> --- src/backend/access/brin/brin_pageops.c | 2 +- src/backend/access/brin/brin_xlog.c | 4 +- src/backend/storage/page/bufpage.c | 194 +++++++++---------------- src/include/storage/bufpage.h | 3 +- 4 files changed, 74 insertions(+), 129 deletions(-) diff --git a/src/backend/access/brin/brin_pageops.c b/src/backend/access/brin/brin_pageops.c index 24ae38be65..68110090f7 100644 --- a/src/backend/access/brin/brin_pageops.c +++ b/src/backend/access/brin/brin_pageops.c @@ -245,7 +245,7 @@ brin_doupdate(Relation idxrel, BlockNumber pagesPerRange, if (extended) brin_page_init(BufferGetPage(newbuf), BRIN_PAGETYPE_REGULAR); - PageIndexDeleteNoCompact(oldpage, &oldoff, 1); + PageIndexTupleDeleteNoCompact(oldpage, oldoff); newoff = PageAddItem(newpage, (Item) newtup, newsz, InvalidOffsetNumber, false, false); if (newoff == InvalidOffsetNumber) diff --git a/src/backend/access/brin/brin_xlog.c b/src/backend/access/brin/brin_xlog.c index ed14729955..5a6b7288a9 100644 --- a/src/backend/access/brin/brin_xlog.c +++ b/src/backend/access/brin/brin_xlog.c @@ -148,10 +148,8 @@ brin_xlog_update(XLogReaderState *record) page = (Page) BufferGetPage(buffer); offnum = xlrec->oldOffnum; - if (PageGetMaxOffsetNumber(page) + 1 < offnum) - elog(PANIC, "brin_xlog_update: invalid max offset number"); - PageIndexDeleteNoCompact(page, &offnum, 1); + PageIndexTupleDeleteNoCompact(page, offnum); PageSetLSN(page, lsn); MarkBufferDirty(buffer); diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index 41a7dd0d62..73aa0c012c 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -415,8 +415,7 @@ PageRestoreTempPage(Page tempPage, Page oldPage) } /* - * sorting support for PageRepairFragmentation, PageIndexMultiDelete, - * PageIndexDeleteNoCompact + * sorting support for PageRepairFragmentation and PageIndexMultiDelete */ typedef struct itemIdSortData { @@ -762,15 +761,14 @@ PageIndexTupleDelete(Page page, OffsetNumber offnum) * Now move everything between the old upper bound (beginning of tuple * space) and the beginning of the deleted tuple forward, so that space in * the middle of the page is left free. If we've just deleted the tuple - * at the beginning of tuple space, then there's no need to do the copy - * (and bcopy on some architectures SEGV's if asked to move zero bytes). + * at the beginning of tuple space, then there's no need to do the copy. */ /* beginning of tuple space */ addr = (char *) page + phdr->pd_upper; if (offset > phdr->pd_upper) - memmove(addr + size, addr, (int) (offset - phdr->pd_upper)); + memmove(addr + size, addr, offset - phdr->pd_upper); /* adjust free space boundary pointers */ phdr->pd_upper += size; @@ -918,155 +916,105 @@ PageIndexMultiDelete(Page page, OffsetNumber *itemnos, int nitems) compactify_tuples(itemidbase, nused, page); } + /* - * PageIndexDeleteNoCompact - * Delete the given items for an index page, and defragment the resulting - * free space, but do not compact the item pointers array. - * - * itemnos is the array of tuples to delete; nitems is its size. maxIdxTuples - * is the maximum number of tuples that can exist in a page. + * PageIndexTupleDeleteNoCompact * - * Unused items at the end of the array are removed. + * Remove the specified tuple from an index page, but set its line pointer + * to "unused" instead of compacting it out, except that it can be removed + * if it's the last line pointer on the page. * * This is used for index AMs that require that existing TIDs of live tuples - * remain unchanged. + * remain unchanged, and are willing to allow unused line pointers instead. */ void -PageIndexDeleteNoCompact(Page page, OffsetNumber *itemnos, int nitems) +PageIndexTupleDeleteNoCompact(Page page, OffsetNumber offnum) { PageHeader phdr = (PageHeader) page; - LocationIndex pd_lower = phdr->pd_lower; - LocationIndex pd_upper = phdr->pd_upper; - LocationIndex pd_special = phdr->pd_special; + char *addr; + ItemId tup; + Size size; + unsigned offset; int nline; - bool empty; - OffsetNumber offnum; - int nextitm; /* * As with PageRepairFragmentation, paranoia seems justified. */ - if (pd_lower < SizeOfPageHeaderData || - pd_lower > pd_upper || - pd_upper > pd_special || - pd_special > BLCKSZ || - pd_special != MAXALIGN(pd_special)) + if (phdr->pd_lower < SizeOfPageHeaderData || + phdr->pd_lower > phdr->pd_upper || + phdr->pd_upper > phdr->pd_special || + phdr->pd_special > BLCKSZ || + phdr->pd_special != MAXALIGN(phdr->pd_special)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupted page pointers: lower = %u, upper = %u, special = %u", - pd_lower, pd_upper, pd_special))); + phdr->pd_lower, phdr->pd_upper, phdr->pd_special))); + + nline = PageGetMaxOffsetNumber(page); + if ((int) offnum <= 0 || (int) offnum > nline) + elog(ERROR, "invalid index offnum: %u", offnum); + + tup = PageGetItemId(page, offnum); + Assert(ItemIdHasStorage(tup)); + size = ItemIdGetLength(tup); + offset = ItemIdGetOffset(tup); + + if (offset < phdr->pd_upper || (offset + size) > phdr->pd_special || + offset != MAXALIGN(offset)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("corrupted item pointer: offset = %u, size = %u", + offset, (unsigned int) size))); + + /* Amount of space to actually be deleted */ + size = MAXALIGN(size); /* - * Scan the existing item pointer array and mark as unused those that are - * in our kill-list; make sure any non-interesting ones are marked unused - * as well. + * Either set the item pointer to "unused", or zap it if it's the last + * one. (Note: it's possible that the next-to-last one(s) are already + * unused, but we do not trouble to try to compact them out if so.) */ - nline = PageGetMaxOffsetNumber(page); - empty = true; - nextitm = 0; - for (offnum = FirstOffsetNumber; offnum <= nline; offnum = OffsetNumberNext(offnum)) + if ((int) offnum < nline) + ItemIdSetUnused(tup); + else { - ItemId lp; - ItemLength itemlen; - ItemOffset offset; + phdr->pd_lower -= sizeof(ItemIdData); + nline--; /* there's one less than when we started */ + } - lp = PageGetItemId(page, offnum); + /* + * Now move everything between the old upper bound (beginning of tuple + * space) and the beginning of the deleted tuple forward, so that space in + * the middle of the page is left free. If we've just deleted the tuple + * at the beginning of tuple space, then there's no need to do the copy. + */ - itemlen = ItemIdGetLength(lp); - offset = ItemIdGetOffset(lp); + /* beginning of tuple space */ + addr = (char *) page + phdr->pd_upper; - if (ItemIdIsUsed(lp)) - { - if (offset < pd_upper || - (offset + itemlen) > pd_special || - offset != MAXALIGN(offset)) - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("corrupted item pointer: offset = %u, length = %u", - offset, (unsigned int) itemlen))); - - if (nextitm < nitems && offnum == itemnos[nextitm]) - { - /* this one is on our list to delete, so mark it unused */ - ItemIdSetUnused(lp); - nextitm++; - } - else if (ItemIdHasStorage(lp)) - { - /* This one's live -- must do the compaction dance */ - empty = false; - } - else - { - /* get rid of this one too */ - ItemIdSetUnused(lp); - } - } - } + if (offset > phdr->pd_upper) + memmove(addr + size, addr, offset - phdr->pd_upper); - /* this will catch invalid or out-of-order itemnos[] */ - if (nextitm != nitems) - elog(ERROR, "incorrect index offsets supplied"); + /* adjust free space boundary pointer */ + phdr->pd_upper += size; - if (empty) - { - /* Page is completely empty, so just reset it quickly */ - phdr->pd_lower = SizeOfPageHeaderData; - phdr->pd_upper = pd_special; - } - else + /* + * Finally, we need to adjust the linp entries that remain. + * + * Anything that used to be before the deleted tuple's data was moved + * forward by the size of the deleted tuple. + */ + if (!PageIsEmpty(page)) { - /* There are live items: need to compact the page the hard way */ - itemIdSortData itemidbase[MaxOffsetNumber]; - itemIdSort itemidptr; int i; - Size totallen; - /* - * Scan the page taking note of each item that we need to preserve. - * This includes both live items (those that contain data) and - * interspersed unused ones. It's critical to preserve these unused - * items, because otherwise the offset numbers for later live items - * would change, which is not acceptable. Unused items might get used - * again later; that is fine. - */ - itemidptr = itemidbase; - totallen = 0; - PageClearHasFreeLinePointers(page); - for (i = 0; i < nline; i++) + for (i = 1; i <= nline; i++) { - ItemId lp; - - itemidptr->offsetindex = i; + ItemId ii = PageGetItemId(phdr, i); - lp = PageGetItemId(page, i + 1); - if (ItemIdHasStorage(lp)) - { - itemidptr->itemoff = ItemIdGetOffset(lp); - itemidptr->alignedlen = MAXALIGN(ItemIdGetLength(lp)); - totallen += itemidptr->alignedlen; - itemidptr++; - } - else - { - PageSetHasFreeLinePointers(page); - ItemIdSetUnused(lp); - } + if (ItemIdHasStorage(ii) && ItemIdGetOffset(ii) <= offset) + ii->lp_off += size; } - nline = itemidptr - itemidbase; - /* By here, there are exactly nline elements in itemidbase array */ - - if (totallen > (Size) (pd_special - pd_lower)) - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("corrupted item lengths: total %u, available space %u", - (unsigned int) totallen, pd_special - pd_lower))); - - /* - * Defragment the data areas of each tuple, being careful to preserve - * each item's position in the linp array. - */ - compactify_tuples(itemidbase, nline, page); } } diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h index 4cedff7fa4..ad4ab5f7fc 100644 --- a/src/include/storage/bufpage.h +++ b/src/include/storage/bufpage.h @@ -429,8 +429,7 @@ extern Size PageGetExactFreeSpace(Page page); extern Size PageGetHeapFreeSpace(Page page); extern void PageIndexTupleDelete(Page page, OffsetNumber offset); extern void PageIndexMultiDelete(Page page, OffsetNumber *itemnos, int nitems); -extern void PageIndexDeleteNoCompact(Page page, OffsetNumber *itemnos, - int nitems); +extern void PageIndexTupleDeleteNoCompact(Page page, OffsetNumber offset); extern bool PageIndexTupleOverwrite(Page page, OffsetNumber offnum, Item newtup, Size newsize); extern char *PageSetChecksumCopy(Page page, BlockNumber blkno); From ddc889317912fd8b654439701195a43cecfd4e79 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 10 Sep 2016 13:49:04 -0400 Subject: [PATCH 150/871] Fix miserable coding in pg_stat_get_activity(). Commit dd1a3bccc replaced a test on whether a subroutine returned a null pointer with a test on whether &pointer->backendStatus was null. This accidentally failed to fail, at least on common compilers, because backendStatus is the first field in the struct; but it was surely trouble waiting to happen. Commit f91feba87 then messed things up further, changing the logic to local_beentry = pgstat_fetch_stat_local_beentry(curr_backend); if (!local_beentry) continue; beentry = &local_beentry->backendStatus; if (!beentry) { where the second "if" is now dead code, so that the intended behavior of printing a row with "" cannot occur. I suspect this is all moot because pgstat_fetch_stat_local_beentry will never actually return null in this function's usage, but it's still very poor coding. Repair back to 9.4 where the original problem was introduced. --- src/backend/utils/adt/pgstatfuncs.c | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 5d1ccf51d4..2d3cf9e468 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -688,27 +688,17 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) MemSet(values, 0, sizeof(values)); MemSet(nulls, 0, sizeof(nulls)); - if (pid != -1) - { - /* Skip any which are not the one we're looking for. */ - PgBackendStatus *be = pgstat_fetch_stat_beentry(curr_backend); - - if (!be || be->st_procpid != pid) - continue; - - } - /* Get the next one in the list */ local_beentry = pgstat_fetch_stat_local_beentry(curr_backend); if (!local_beentry) - continue; - - beentry = &local_beentry->backendStatus; - if (!beentry) { int i; - for (i = 0; i < sizeof(nulls) / sizeof(nulls[0]); i++) + /* Ignore missing entries if looking for specific PID */ + if (pid != -1) + continue; + + for (i = 0; i < lengthof(nulls); i++) nulls[i] = true; nulls[5] = false; @@ -718,6 +708,12 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) continue; } + beentry = &local_beentry->backendStatus; + + /* If looking for specific PID, ignore all the others */ + if (pid != -1 && beentry->st_procpid != pid) + continue; + /* Values available to all callers */ values[0] = ObjectIdGetDatum(beentry->st_databaseid); values[1] = Int32GetDatum(beentry->st_procpid); From f2717c79eeecaf5997016d52fd81881301dcfc5e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 10 Sep 2016 17:54:23 -0400 Subject: [PATCH 151/871] Improve unreachability recognition in elog() macro. Some experimentation with an older version of gcc showed that it is able to determine whether "if (elevel_ >= ERROR)" is compile-time constant if elevel_ is declared "const", but otherwise not so much. We had accounted for that in ereport() but were too miserly with braces to make it so in elog(). I don't know how many currently-interesting compilers have the same quirk, but in case it will save some code space, let's make sure that elog() is on the same footing as ereport() for this purpose. Back-patch to 9.3 where we introduced pg_unreachable() calls into elog/ereport. --- src/include/utils/elog.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index f4ff03ec8a..70dc365465 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -206,12 +206,13 @@ extern int getinternalerrposition(void); #else /* !HAVE__BUILTIN_CONSTANT_P */ #define elog(elevel, ...) \ do { \ - int elevel_; \ elog_start(__FILE__, __LINE__, PG_FUNCNAME_MACRO); \ - elevel_ = (elevel); \ - elog_finish(elevel_, __VA_ARGS__); \ - if (elevel_ >= ERROR) \ - pg_unreachable(); \ + { \ + const int elevel_ = (elevel); \ + elog_finish(elevel_, __VA_ARGS__); \ + if (elevel_ >= ERROR) \ + pg_unreachable(); \ + } \ } while(0) #endif /* HAVE__BUILTIN_CONSTANT_P */ #else /* !HAVE__VA_ARGS */ From 24598337c8d214ba8dcf354130b72c49636bba69 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sun, 11 Sep 2016 16:27:27 +0300 Subject: [PATCH 152/871] Implement binary heap replace-top operation in a smarter way. In external sort's merge phase, we maintain a binary heap holding the next tuple from each input tape. On each step, the topmost tuple is returned, and replaced with the next tuple from the same tape. We were doing the replacement by deleting the top node in one operation, and inserting the next tuple after that. However, you can do a "replace-top" operation more efficiently, in one "sift-up". A deletion will always walk the heap from top to bottom, but in a replacement, we can stop as soon as we find the right place for the new tuple. This is particularly helpful, if the tapes are not in completely random order, so that the next tuple from a tape is likely to land near the top of the heap. Peter Geoghegan, reviewed by Claudio Freire, with some editing by me. Discussion: --- src/backend/utils/sort/tuplesort.c | 182 +++++++++++++++++------------ 1 file changed, 109 insertions(+), 73 deletions(-) diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index aa8e0e42fc..d600670d26 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -69,25 +69,25 @@ * using Algorithm D. * * When merging runs, we use a heap containing just the frontmost tuple from - * each source run; we repeatedly output the smallest tuple and insert the - * next tuple from its source tape (if any). When the heap empties, the merge - * is complete. The basic merge algorithm thus needs very little memory --- - * only M tuples for an M-way merge, and M is constrained to a small number. - * However, we can still make good use of our full workMem allocation by - * pre-reading additional tuples from each source tape. Without prereading, - * our access pattern to the temporary file would be very erratic; on average - * we'd read one block from each of M source tapes during the same time that - * we're writing M blocks to the output tape, so there is no sequentiality of - * access at all, defeating the read-ahead methods used by most Unix kernels. - * Worse, the output tape gets written into a very random sequence of blocks - * of the temp file, ensuring that things will be even worse when it comes - * time to read that tape. A straightforward merge pass thus ends up doing a - * lot of waiting for disk seeks. We can improve matters by prereading from - * each source tape sequentially, loading about workMem/M bytes from each tape - * in turn. Then we run the merge algorithm, writing but not reading until - * one of the preloaded tuple series runs out. Then we switch back to preread - * mode, fill memory again, and repeat. This approach helps to localize both - * read and write accesses. + * each source run; we repeatedly output the smallest tuple and replace it + * with the next tuple from its source tape (if any). When the heap empties, + * the merge is complete. The basic merge algorithm thus needs very little + * memory --- only M tuples for an M-way merge, and M is constrained to a + * small number. However, we can still make good use of our full workMem + * allocation by pre-reading additional tuples from each source tape. Without + * prereading, our access pattern to the temporary file would be very erratic; + * on average we'd read one block from each of M source tapes during the same + * time that we're writing M blocks to the output tape, so there is no + * sequentiality of access at all, defeating the read-ahead methods used by + * most Unix kernels. Worse, the output tape gets written into a very random + * sequence of blocks of the temp file, ensuring that things will be even + * worse when it comes time to read that tape. A straightforward merge pass + * thus ends up doing a lot of waiting for disk seeks. We can improve matters + * by prereading from each source tape sequentially, loading about workMem/M + * bytes from each tape in turn. Then we run the merge algorithm, writing but + * not reading until one of the preloaded tuple series runs out. Then we + * switch back to preread mode, fill memory again, and repeat. This approach + * helps to localize both read and write accesses. * * When the caller requests random access to the sort result, we form * the final sorted run on a logical tape which is then "frozen", so @@ -569,8 +569,10 @@ static void make_bounded_heap(Tuplesortstate *state); static void sort_bounded_heap(Tuplesortstate *state); static void tuplesort_sort_memtuples(Tuplesortstate *state); static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple, - int tupleindex, bool checkIndex); -static void tuplesort_heap_siftup(Tuplesortstate *state, bool checkIndex); + bool checkIndex); +static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple, + bool checkIndex); +static void tuplesort_heap_delete_top(Tuplesortstate *state, bool checkIndex); static void reversedirection(Tuplesortstate *state); static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK); static void markrunend(Tuplesortstate *state, int tapenum); @@ -1617,10 +1619,10 @@ puttuple_common(Tuplesortstate *state, SortTuple *tuple) } else { - /* discard top of heap, sift up, insert new tuple */ + /* discard top of heap, replacing it with the new tuple */ free_sort_tuple(state, &state->memtuples[0]); - tuplesort_heap_siftup(state, false); - tuplesort_heap_insert(state, tuple, 0, false); + tuple->tupindex = 0; /* not used */ + tuplesort_heap_replace_top(state, tuple, false); } break; @@ -1665,7 +1667,8 @@ puttuple_common(Tuplesortstate *state, SortTuple *tuple) * initial COMPARETUP() call is required for the tuple, to * determine that the tuple does not belong in RUN_FIRST). */ - tuplesort_heap_insert(state, tuple, state->currentRun, true); + tuple->tupindex = state->currentRun; + tuplesort_heap_insert(state, tuple, true); } else { @@ -1987,7 +1990,6 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward, * more generally. */ *stup = state->memtuples[0]; - tuplesort_heap_siftup(state, false); if ((tupIndex = state->mergenext[srcTape]) == 0) { /* @@ -2008,18 +2010,25 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward, */ if ((tupIndex = state->mergenext[srcTape]) == 0) { + /* Remove the top node from the heap */ + tuplesort_heap_delete_top(state, false); /* Free tape's buffer, avoiding dangling pointer */ if (state->batchUsed) mergebatchfreetape(state, srcTape, stup, should_free); return true; } } - /* pull next preread tuple from list, insert in heap */ + + /* + * pull next preread tuple from list, and replace the returned + * tuple at top of the heap with it. + */ newtup = &state->memtuples[tupIndex]; state->mergenext[srcTape] = newtup->tupindex; if (state->mergenext[srcTape] == 0) state->mergelast[srcTape] = 0; - tuplesort_heap_insert(state, newtup, srcTape, false); + newtup->tupindex = srcTape; + tuplesort_heap_replace_top(state, newtup, false); /* put the now-unused memtuples entry on the freelist */ newtup->tupindex = state->mergefreelist; state->mergefreelist = tupIndex; @@ -2394,7 +2403,8 @@ inittapes(Tuplesortstate *state) /* Must copy source tuple to avoid possible overwrite */ SortTuple stup = state->memtuples[j]; - tuplesort_heap_insert(state, &stup, 0, false); + stup.tupindex = RUN_FIRST; + tuplesort_heap_insert(state, &stup, false); } Assert(state->memtupcount == ntuples); } @@ -2642,22 +2652,29 @@ mergeonerun(Tuplesortstate *state) /* writetup adjusted total free space, now fix per-tape space */ spaceFreed = state->availMem - priorAvail; state->mergeavailmem[srcTape] += spaceFreed; - /* compact the heap */ - tuplesort_heap_siftup(state, false); if ((tupIndex = state->mergenext[srcTape]) == 0) { /* out of preloaded data on this tape, try to read more */ mergepreread(state); /* if still no data, we've reached end of run on this tape */ if ((tupIndex = state->mergenext[srcTape]) == 0) + { + /* remove the written-out tuple from the heap */ + tuplesort_heap_delete_top(state, false); continue; + } } - /* pull next preread tuple from list, insert in heap */ + + /* + * pull next preread tuple from list, and replace the written-out + * tuple in the heap with it. + */ tup = &state->memtuples[tupIndex]; state->mergenext[srcTape] = tup->tupindex; if (state->mergenext[srcTape] == 0) state->mergelast[srcTape] = 0; - tuplesort_heap_insert(state, tup, srcTape, false); + tup->tupindex = srcTape; + tuplesort_heap_replace_top(state, tup, false); /* put the now-unused memtuples entry on the freelist */ tup->tupindex = state->mergefreelist; state->mergefreelist = tupIndex; @@ -2793,7 +2810,8 @@ beginmerge(Tuplesortstate *state, bool finalMergeBatch) state->mergenext[srcTape] = tup->tupindex; if (state->mergenext[srcTape] == 0) state->mergelast[srcTape] = 0; - tuplesort_heap_insert(state, tup, srcTape, false); + tup->tupindex = srcTape; + tuplesort_heap_insert(state, tup, false); /* put the now-unused memtuples entry on the freelist */ tup->tupindex = state->mergefreelist; state->mergefreelist = tupIndex; @@ -3275,13 +3293,12 @@ dumptuples(Tuplesortstate *state, bool alltuples) * Still holding out for a case favorable to replacement * selection. Still incrementally spilling using heap. * - * Dump the heap's frontmost entry, and sift up to remove it from - * the heap. + * Dump the heap's frontmost entry, and remove it from the heap. */ Assert(state->memtupcount > 0); WRITETUP(state, state->tp_tapenum[state->destTape], &state->memtuples[0]); - tuplesort_heap_siftup(state, true); + tuplesort_heap_delete_top(state, true); } else { @@ -3644,27 +3661,29 @@ make_bounded_heap(Tuplesortstate *state) state->memtupcount = 0; /* make the heap empty */ for (i = 0; i < tupcount; i++) { - if (state->memtupcount >= state->bound && - COMPARETUP(state, &state->memtuples[i], &state->memtuples[0]) <= 0) - { - /* New tuple would just get thrown out, so skip it */ - free_sort_tuple(state, &state->memtuples[i]); - CHECK_FOR_INTERRUPTS(); - } - else + if (state->memtupcount < state->bound) { /* Insert next tuple into heap */ /* Must copy source tuple to avoid possible overwrite */ SortTuple stup = state->memtuples[i]; - tuplesort_heap_insert(state, &stup, 0, false); - - /* If heap too full, discard largest entry */ - if (state->memtupcount > state->bound) + stup.tupindex = 0; /* not used */ + tuplesort_heap_insert(state, &stup, false); + } + else + { + /* + * The heap is full. Replace the largest entry with the new + * tuple, or just discard it, if it's larger than anything already + * in the heap. + */ + if (COMPARETUP(state, &state->memtuples[i], &state->memtuples[0]) <= 0) { - free_sort_tuple(state, &state->memtuples[0]); - tuplesort_heap_siftup(state, false); + free_sort_tuple(state, &state->memtuples[i]); + CHECK_FOR_INTERRUPTS(); } + else + tuplesort_heap_replace_top(state, &state->memtuples[i], false); } } @@ -3685,16 +3704,16 @@ sort_bounded_heap(Tuplesortstate *state) Assert(tupcount == state->bound); /* - * We can unheapify in place because each sift-up will remove the largest - * entry, which we can promptly store in the newly freed slot at the end. - * Once we're down to a single-entry heap, we're done. + * We can unheapify in place because each delete-top call will remove the + * largest entry, which we can promptly store in the newly freed slot at + * the end. Once we're down to a single-entry heap, we're done. */ while (state->memtupcount > 1) { SortTuple stup = state->memtuples[0]; /* this sifts-up the next-largest entry and decreases memtupcount */ - tuplesort_heap_siftup(state, false); + tuplesort_heap_delete_top(state, false); state->memtuples[state->memtupcount] = stup; } state->memtupcount = tupcount; @@ -3738,30 +3757,21 @@ tuplesort_sort_memtuples(Tuplesortstate *state) * Insert a new tuple into an empty or existing heap, maintaining the * heap invariant. Caller is responsible for ensuring there's room. * - * Note: we assume *tuple is a temporary variable that can be scribbled on. - * For some callers, tuple actually points to a memtuples[] entry above the + * Note: For some callers, tuple points to a memtuples[] entry above the * end of the heap. This is safe as long as it's not immediately adjacent * to the end of the heap (ie, in the [memtupcount] array entry) --- if it * is, it might get overwritten before being moved into the heap! */ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple, - int tupleindex, bool checkIndex) + bool checkIndex) { SortTuple *memtuples; int j; - /* - * Save the tupleindex --- see notes above about writing on *tuple. It's a - * historical artifact that tupleindex is passed as a separate argument - * and not in *tuple, but it's notationally convenient so let's leave it - * that way. - */ - tuple->tupindex = tupleindex; - memtuples = state->memtuples; Assert(state->memtupcount < state->memtupsize); - Assert(!checkIndex || tupleindex == RUN_FIRST); + Assert(!checkIndex || tuple->tupindex == RUN_FIRST); CHECK_FOR_INTERRUPTS(); @@ -3783,25 +3793,51 @@ tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple, } /* - * The tuple at state->memtuples[0] has been removed from the heap. - * Decrement memtupcount, and sift up to maintain the heap invariant. + * Remove the tuple at state->memtuples[0] from the heap. Decrement + * memtupcount, and sift up to maintain the heap invariant. + * + * The caller has already free'd the tuple the top node points to, + * if necessary. */ static void -tuplesort_heap_siftup(Tuplesortstate *state, bool checkIndex) +tuplesort_heap_delete_top(Tuplesortstate *state, bool checkIndex) { SortTuple *memtuples = state->memtuples; SortTuple *tuple; - int i, - n; Assert(!checkIndex || state->currentRun == RUN_FIRST); if (--state->memtupcount <= 0) return; + /* + * Remove the last tuple in the heap, and re-insert it, by replacing the + * current top node with it. + */ + tuple = &memtuples[state->memtupcount]; + tuplesort_heap_replace_top(state, tuple, checkIndex); +} + +/* + * Replace the tuple at state->memtuples[0] with a new tuple. Sift up to + * maintain the heap invariant. + * + * This corresponds to Knuth's "sift-up" algorithm (Algorithm 5.2.3H, + * Heapsort, steps H3-H8). + */ +static void +tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple, + bool checkIndex) +{ + SortTuple *memtuples = state->memtuples; + int i, + n; + + Assert(!checkIndex || state->currentRun == RUN_FIRST); + Assert(state->memtupcount >= 1); + CHECK_FOR_INTERRUPTS(); n = state->memtupcount; - tuple = &memtuples[n]; /* tuple that must be reinserted */ i = 0; /* i is where the "hole" is */ for (;;) { From 28e5e5648cc3666537c393b2636c4aa34fdb22c1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 11 Sep 2016 12:46:55 -0400 Subject: [PATCH 153/871] Fix and simplify MSVC build's handling of xml/xslt/uuid dependencies. Solution.pm mistakenly believed that the xml option requires the xslt option, when actually the dependency is the other way around; and it believed that libxml requires libiconv, which is not necessarily so, so we shouldn't enforce it here. Fix the option cross-checking logic. Also, since AddProject already takes care of adding libxml and libxslt include and library dependencies to every project, there's no need for the custom code that did that in mkvcbuild. While at it, let's handle the similar dependencies for uuid in a similar fashion. Given the lack of field complaints about these overly strict build dependency requirements, there seems no need for a back-patch. Michael Paquier Discussion: --- src/tools/msvc/Mkvcbuild.pm | 22 ++-------------------- src/tools/msvc/Solution.pm | 12 +++++++----- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index b3ed1f56e2..93dfd24a83 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -381,18 +381,7 @@ sub mkvcbuild $zic->AddDirResourceFile('src/timezone'); $zic->AddReference($libpgcommon, $libpgport); - if ($solution->{options}->{xml}) - { - $contrib_extraincludes->{'pgxml'} = [ - $solution->{options}->{xml} . '/include', - $solution->{options}->{xslt} . '/include', - $solution->{options}->{iconv} . '/include' ]; - - $contrib_extralibs->{'pgxml'} = [ - $solution->{options}->{xml} . '/lib/libxml2.lib', - $solution->{options}->{xslt} . '/lib/libxslt.lib' ]; - } - else + if (!$solution->{options}->{xml}) { push @contrib_excludes, 'xml2'; } @@ -402,14 +391,7 @@ sub mkvcbuild push @contrib_excludes, 'sslinfo'; } - if ($solution->{options}->{uuid}) - { - $contrib_extraincludes->{'uuid-ossp'} = - [ $solution->{options}->{uuid} . '/include' ]; - $contrib_extralibs->{'uuid-ossp'} = - [ $solution->{options}->{uuid} . '/lib/uuid.lib' ]; - } - else + if (!$solution->{options}->{uuid}) { push @contrib_excludes, 'uuid-ossp'; } diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index 9cb1ad36cf..8217d06f28 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -37,12 +37,9 @@ sub _new unless exists $options->{float8byval}; die "float8byval not permitted on 32 bit platforms" if $options->{float8byval} && $bits == 32; - if ($options->{xml}) + if ($options->{xslt} && !$options->{xml}) { - if (!($options->{xslt} && $options->{iconv})) - { - die "XML requires both XSLT and ICONV\n"; - } + die "XSLT requires XML\n"; } $options->{blocksize} = 8 unless $options->{blocksize}; # undef or 0 means default @@ -555,6 +552,11 @@ sub AddProject $proj->AddIncludeDir($self->{options}->{xslt} . '\include'); $proj->AddLibrary($self->{options}->{xslt} . '\lib\libxslt.lib'); } + if ($self->{options}->{uuid}) + { + $proj->AddIncludeDir($self->{options}->{uuid} . '\include'); + $proj->AddLibrary($self->{options}->{uuid} . '\lib\uuid.lib'); + } return $proj; } From 40b449ae84dcf71177d7749a7b0c582b64dc15f0 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 11 Sep 2016 14:15:07 -0400 Subject: [PATCH 154/871] Allow CREATE EXTENSION to follow extension update paths. Previously, to update an extension you had to produce both a version-update script and a new base installation script. It's become more and more obvious that that's tedious, duplicative, and error-prone. This patch attempts to improve matters by allowing the new base installation script to be omitted. CREATE EXTENSION will install a requested version if it can find a base script and a chain of update scripts that will get there. As in the existing update logic, shorter chains are preferred if there's more than one possibility, with an arbitrary tie-break rule for chains of equal length. Also adjust the pg_available_extension_versions view to show such versions as installable. While at it, refactor the code so that CASCADE processing works for extensions requested during ApplyExtensionUpdates(). Without this, addition of a new requirement in an updated extension would require creating a new base script, even if there was no other reason to do that. (It would be easy at this point to add a CASCADE option to ALTER EXTENSION UPDATE, to allow the same thing to happen during a manually-commanded version update, but I have not done that here.) Tom Lane, reviewed by Andres Freund Discussion: <20160905005919.jz2m2yh3und2dsuy@alap3.anarazel.de> --- doc/src/sgml/extend.sgml | 41 +++ src/backend/commands/extension.c | 608 ++++++++++++++++++++----------- 2 files changed, 439 insertions(+), 210 deletions(-) diff --git a/doc/src/sgml/extend.sgml b/doc/src/sgml/extend.sgml index df88380a23..e19c657d8f 100644 --- a/doc/src/sgml/extend.sgml +++ b/doc/src/sgml/extend.sgml @@ -885,6 +885,47 @@ SELECT * FROM pg_extension_update_paths('extension_name'); + + Installing Extensions using Update Scripts + + + An extension that has been around for awhile will probably exist in + several versions, for which the author will need to write update scripts. + For example, if you have released a foo extension in + versions 1.0, 1.1, and 1.2, there + should be update scripts foo--1.0--1.1.sql + and foo--1.1--1.2.sql. + Before PostgreSQL 10, it was necessary to also create + new script files foo--1.1.sql and foo--1.2.sql + that directly build the newer extension versions, or else the newer + versions could not be installed directly, only by + installing 1.0 and then updating. That was tedious and + duplicative, but now it's unnecessary, because CREATE + EXTENSION can follow update chains automatically. + For example, if only the script + files foo--1.0.sql, foo--1.0--1.1.sql, + and foo--1.1--1.2.sql are available then a request to + install version 1.2 is honored by running those three + scripts in sequence. The processing is the same as if you'd first + installed 1.0 and then updated to 1.2. + (As with ALTER EXTENSION UPDATE, if multiple pathways are + available then the shortest is preferred.) Arranging an extension's + script files in this style can reduce the amount of maintenance effort + needed to produce small updates. + + + + If you use secondary (version-specific) control files with an extension + maintained in this style, keep in mind that each version needs a control + file even if it has no stand-alone installation script, as that control + file will determine how the implicit update to that version is performed. + For example, if foo--1.0.control specifies requires + = 'bar' but foo's other control files do not, the + extension's dependency on bar will be dropped when updating + from 1.0 to another version. + + + Extension Example diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c index df49a78e2f..f6c2c8af91 100644 --- a/src/backend/commands/extension.c +++ b/src/backend/commands/extension.c @@ -100,14 +100,25 @@ typedef struct ExtensionVersionInfo static List *find_update_path(List *evi_list, ExtensionVersionInfo *evi_start, ExtensionVersionInfo *evi_target, + bool reject_indirect, bool reinitialize); +static Oid get_required_extension(char *reqExtensionName, + char *extensionName, + char *origSchemaName, + bool cascade, + List *parents, + bool is_create); static void get_available_versions_for_extension(ExtensionControlFile *pcontrol, Tuplestorestate *tupstore, TupleDesc tupdesc); +static Datum convert_requires_to_datum(List *requires); static void ApplyExtensionUpdates(Oid extensionOid, ExtensionControlFile *pcontrol, const char *initialVersion, - List *updateVersions); + List *updateVersions, + char *origSchemaName, + bool cascade, + bool is_create); static char *read_whole_file(const char *filename, int *length); @@ -1071,7 +1082,7 @@ identify_update_path(ExtensionControlFile *control, evi_target = get_ext_ver_info(newVersion, &evi_list); /* Find shortest path */ - result = find_update_path(evi_list, evi_start, evi_target, false); + result = find_update_path(evi_list, evi_start, evi_target, false, false); if (result == NIL) ereport(ERROR, @@ -1086,9 +1097,13 @@ identify_update_path(ExtensionControlFile *control, * Apply Dijkstra's algorithm to find the shortest path from evi_start to * evi_target. * + * If reject_indirect is true, ignore paths that go through installable + * versions. This saves work when the caller will consider starting from + * all installable versions anyway. + * * If reinitialize is false, assume the ExtensionVersionInfo list has not * been used for this before, and the initialization done by get_ext_ver_info - * is still good. + * is still good. Otherwise, reinitialize all transient fields used here. * * Result is a List of names of versions to transition through (the initial * version is *not* included). Returns NIL if no such path. @@ -1097,6 +1112,7 @@ static List * find_update_path(List *evi_list, ExtensionVersionInfo *evi_start, ExtensionVersionInfo *evi_target, + bool reject_indirect, bool reinitialize) { List *result; @@ -1105,6 +1121,8 @@ find_update_path(List *evi_list, /* Caller error if start == target */ Assert(evi_start != evi_target); + /* Caller error if reject_indirect and target is installable */ + Assert(!(reject_indirect && evi_target->installable)); if (reinitialize) { @@ -1131,6 +1149,9 @@ find_update_path(List *evi_list, ExtensionVersionInfo *evi2 = (ExtensionVersionInfo *) lfirst(lc); int newdist; + /* if reject_indirect, treat installable versions as unreachable */ + if (reject_indirect && evi2->installable) + continue; newdist = evi->distance + 1; if (newdist < evi2->distance) { @@ -1166,6 +1187,67 @@ find_update_path(List *evi_list, return result; } +/* + * Given a target version that is not directly installable, find the + * best installation sequence starting from a directly-installable version. + * + * evi_list: previously-collected version update graph + * evi_target: member of that list that we want to reach + * + * Returns the best starting-point version, or NULL if there is none. + * On success, *best_path is set to the path from the start point. + * + * If there's more than one possible start point, prefer shorter update paths, + * and break any ties arbitrarily on the basis of strcmp'ing the starting + * versions' names. + */ +static ExtensionVersionInfo * +find_install_path(List *evi_list, ExtensionVersionInfo *evi_target, + List **best_path) +{ + ExtensionVersionInfo *evi_start = NULL; + ListCell *lc; + + *best_path = NIL; + + /* + * We don't expect to be called for an installable target, but if we are, + * the answer is easy: just start from there, with an empty update path. + */ + if (evi_target->installable) + return evi_target; + + /* Consider all installable versions as start points */ + foreach(lc, evi_list) + { + ExtensionVersionInfo *evi1 = (ExtensionVersionInfo *) lfirst(lc); + List *path; + + if (!evi1->installable) + continue; + + /* + * Find shortest path from evi1 to evi_target; but no need to consider + * paths going through other installable versions. + */ + path = find_update_path(evi_list, evi1, evi_target, true, true); + if (path == NIL) + continue; + + /* Remember best path */ + if (evi_start == NULL || + list_length(path) < list_length(*best_path) || + (list_length(path) == list_length(*best_path) && + strcmp(evi_start->name, evi1->name) < 0)) + { + evi_start = evi1; + *best_path = path; + } + } + + return evi_start; +} + /* * CREATE EXTENSION worker * @@ -1175,17 +1257,16 @@ find_update_path(List *evi_list, * installed, allowing us to error out if we recurse to one of those. */ static ObjectAddress -CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *parents) +CreateExtensionInternal(char *extensionName, + char *schemaName, + char *versionName, + char *oldVersionName, + bool cascade, + List *parents, + bool is_create) { - DefElem *d_schema = NULL; - DefElem *d_new_version = NULL; - DefElem *d_old_version = NULL; - DefElem *d_cascade = NULL; - char *schemaName = NULL; + char *origSchemaName = schemaName; Oid schemaOid = InvalidOid; - char *versionName; - char *oldVersionName; - bool cascade = false; Oid extowner = GetUserId(); ExtensionControlFile *pcontrol; ExtensionControlFile *control; @@ -1193,87 +1274,43 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par List *requiredExtensions; List *requiredSchemas; Oid extensionOid; - ListCell *lc; ObjectAddress address; + ListCell *lc; /* * Read the primary control file. Note we assume that it does not contain * any non-ASCII data, so there is no need to worry about encoding at this * point. */ - pcontrol = read_extension_control_file(stmt->extname); - - /* - * Read the statement option list - */ - foreach(lc, stmt->options) - { - DefElem *defel = (DefElem *) lfirst(lc); - - if (strcmp(defel->defname, "schema") == 0) - { - if (d_schema) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"), - parser_errposition(pstate, defel->location))); - d_schema = defel; - } - else if (strcmp(defel->defname, "new_version") == 0) - { - if (d_new_version) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"), - parser_errposition(pstate, defel->location))); - d_new_version = defel; - } - else if (strcmp(defel->defname, "old_version") == 0) - { - if (d_old_version) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"), - parser_errposition(pstate, defel->location))); - d_old_version = defel; - } - else if (strcmp(defel->defname, "cascade") == 0) - { - if (d_cascade) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"), - parser_errposition(pstate, defel->location))); - d_cascade = defel; - cascade = defGetBoolean(d_cascade); - } - else - elog(ERROR, "unrecognized option: %s", defel->defname); - } + pcontrol = read_extension_control_file(extensionName); /* * Determine the version to install */ - if (d_new_version && d_new_version->arg) - versionName = strVal(d_new_version->arg); - else if (pcontrol->default_version) - versionName = pcontrol->default_version; - else + if (versionName == NULL) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("version to install must be specified"))); - versionName = NULL; /* keep compiler quiet */ + if (pcontrol->default_version) + versionName = pcontrol->default_version; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("version to install must be specified"))); } check_valid_version_name(versionName); /* - * Determine the (unpackaged) version to update from, if any, and then - * figure out what sequence of update scripts we need to apply. + * Figure out which script(s) we need to run to install the desired + * version of the extension. If we do not have a script that directly + * does what is needed, we try to find a sequence of update scripts that + * will get us there. */ - if (d_old_version && d_old_version->arg) + if (oldVersionName) { - oldVersionName = strVal(d_old_version->arg); + /* + * "FROM old_version" was specified, indicating that we're trying to + * update from some unpackaged version of the extension. Locate a + * series of update scripts that will do it. + */ check_valid_version_name(oldVersionName); if (strcmp(oldVersionName, versionName) == 0) @@ -1308,8 +1345,48 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par } else { + /* + * No FROM, so we're installing from scratch. If there is an install + * script for the desired version, we only need to run that one. + */ + char *filename; + struct stat fst; + oldVersionName = NULL; - updateVersions = NIL; + + filename = get_extension_script_filename(pcontrol, NULL, versionName); + if (stat(filename, &fst) == 0) + { + /* Easy, no extra scripts */ + updateVersions = NIL; + } + else + { + /* Look for best way to install this version */ + List *evi_list; + ExtensionVersionInfo *evi_start; + ExtensionVersionInfo *evi_target; + + /* Extract the version update graph from the script directory */ + evi_list = get_ext_ver_list(pcontrol); + + /* Identify the target version */ + evi_target = get_ext_ver_info(versionName, &evi_list); + + /* Identify best path to reach target */ + evi_start = find_install_path(evi_list, evi_target, + &updateVersions); + + /* Fail if no path ... */ + if (evi_start == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("extension \"%s\" has no installation script nor update path for version \"%s\"", + pcontrol->name, versionName))); + + /* Otherwise, install best starting point and then upgrade */ + versionName = evi_start->name; + } } /* @@ -1320,13 +1397,8 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par /* * Determine the target schema to install the extension into */ - if (d_schema && d_schema->arg) + if (schemaName) { - /* - * User given schema, CREATE EXTENSION ... WITH SCHEMA ... - */ - schemaName = strVal(d_schema->arg); - /* If the user is giving us the schema name, it must exist already. */ schemaOid = get_namespace_oid(schemaName, false); } @@ -1374,7 +1446,7 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par else if (!OidIsValid(schemaOid)) { /* - * Neither user nor author of the extension specified schema, use the + * Neither user nor author of the extension specified schema; use the * current default creation namespace, which is the first explicit * entry in the search_path. */ @@ -1415,66 +1487,12 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par Oid reqext; Oid reqschema; - reqext = get_extension_oid(curreq, true); - if (!OidIsValid(reqext)) - { - if (cascade) - { - /* Must install it. */ - CreateExtensionStmt *ces; - ListCell *lc2; - ObjectAddress addr; - List *cascade_parents; - - /* Check extension name validity before trying to cascade. */ - check_valid_extension_name(curreq); - - /* Check for cyclic dependency between extensions. */ - foreach(lc2, parents) - { - char *pname = (char *) lfirst(lc2); - - if (strcmp(pname, curreq) == 0) - ereport(ERROR, - (errcode(ERRCODE_INVALID_RECURSION), - errmsg("cyclic dependency detected between extensions \"%s\" and \"%s\"", - curreq, stmt->extname))); - } - - ereport(NOTICE, - (errmsg("installing required extension \"%s\"", - curreq))); - - /* Build a CREATE EXTENSION statement to pass down. */ - ces = makeNode(CreateExtensionStmt); - ces->extname = curreq; - ces->if_not_exists = false; - - /* Propagate the CASCADE option. */ - ces->options = list_make1(d_cascade); - - /* Propagate the SCHEMA option if given. */ - if (d_schema && d_schema->arg) - ces->options = lappend(ces->options, d_schema); - - /* Add current extension to list of parents to pass down. */ - cascade_parents = - lappend(list_copy(parents), stmt->extname); - - /* Create the required extension. */ - addr = CreateExtensionInternal(pstate, ces, cascade_parents); - - /* Get its newly-assigned OID. */ - reqext = addr.objectId; - } - else - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("required extension \"%s\" is not installed", - curreq), - errhint("Use CREATE EXTENSION ... CASCADE to install required extensions too."))); - } - + reqext = get_required_extension(curreq, + extensionName, + origSchemaName, + cascade, + parents, + is_create); reqschema = get_extension_schema(reqext); requiredExtensions = lappend_oid(requiredExtensions, reqext); requiredSchemas = lappend_oid(requiredSchemas, reqschema); @@ -1510,17 +1528,100 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par * though a series of ALTER EXTENSION UPDATE commands were given */ ApplyExtensionUpdates(extensionOid, pcontrol, - versionName, updateVersions); + versionName, updateVersions, + origSchemaName, cascade, is_create); return address; } +/* + * Get the OID of an extension listed in "requires", possibly creating it. + */ +static Oid +get_required_extension(char *reqExtensionName, + char *extensionName, + char *origSchemaName, + bool cascade, + List *parents, + bool is_create) +{ + Oid reqExtensionOid; + + reqExtensionOid = get_extension_oid(reqExtensionName, true); + if (!OidIsValid(reqExtensionOid)) + { + if (cascade) + { + /* Must install it. */ + ObjectAddress addr; + List *cascade_parents; + ListCell *lc; + + /* Check extension name validity before trying to cascade. */ + check_valid_extension_name(reqExtensionName); + + /* Check for cyclic dependency between extensions. */ + foreach(lc, parents) + { + char *pname = (char *) lfirst(lc); + + if (strcmp(pname, reqExtensionName) == 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_RECURSION), + errmsg("cyclic dependency detected between extensions \"%s\" and \"%s\"", + reqExtensionName, extensionName))); + } + + ereport(NOTICE, + (errmsg("installing required extension \"%s\"", + reqExtensionName))); + + /* Add current extension to list of parents to pass down. */ + cascade_parents = lappend(list_copy(parents), extensionName); + + /* + * Create the required extension. We propagate the SCHEMA option + * if any, and CASCADE, but no other options. + */ + addr = CreateExtensionInternal(reqExtensionName, + origSchemaName, + NULL, + NULL, + cascade, + cascade_parents, + is_create); + + /* Get its newly-assigned OID. */ + reqExtensionOid = addr.objectId; + } + else + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("required extension \"%s\" is not installed", + reqExtensionName), + is_create ? + errhint("Use CREATE EXTENSION ... CASCADE to install required extensions too.") : 0)); + } + + return reqExtensionOid; +} + /* * CREATE EXTENSION */ ObjectAddress CreateExtension(ParseState *pstate, CreateExtensionStmt *stmt) { + DefElem *d_schema = NULL; + DefElem *d_new_version = NULL; + DefElem *d_old_version = NULL; + DefElem *d_cascade = NULL; + char *schemaName = NULL; + char *versionName = NULL; + char *oldVersionName = NULL; + bool cascade = false; + ListCell *lc; + /* Check extension name validity before any filesystem access */ check_valid_extension_name(stmt->extname); @@ -1556,8 +1657,63 @@ CreateExtension(ParseState *pstate, CreateExtensionStmt *stmt) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("nested CREATE EXTENSION is not supported"))); - /* Finally create the extension. */ - return CreateExtensionInternal(pstate, stmt, NIL); + /* Deconstruct the statement option list */ + foreach(lc, stmt->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "schema") == 0) + { + if (d_schema) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); + d_schema = defel; + schemaName = defGetString(d_schema); + } + else if (strcmp(defel->defname, "new_version") == 0) + { + if (d_new_version) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); + d_new_version = defel; + versionName = defGetString(d_new_version); + } + else if (strcmp(defel->defname, "old_version") == 0) + { + if (d_old_version) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); + d_old_version = defel; + oldVersionName = defGetString(d_old_version); + } + else if (strcmp(defel->defname, "cascade") == 0) + { + if (d_cascade) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); + d_cascade = defel; + cascade = defGetBoolean(d_cascade); + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } + + /* Call CreateExtensionInternal to do the real work. */ + return CreateExtensionInternal(stmt->extname, + schemaName, + versionName, + oldVersionName, + cascade, + NIL, + true); } /* @@ -1914,43 +2070,28 @@ get_available_versions_for_extension(ExtensionControlFile *pcontrol, Tuplestorestate *tupstore, TupleDesc tupdesc) { - int extnamelen = strlen(pcontrol->name); - char *location; - DIR *dir; - struct dirent *de; + List *evi_list; + ListCell *lc; - location = get_extension_script_directory(pcontrol); - dir = AllocateDir(location); - /* Note this will fail if script directory doesn't exist */ - while ((de = ReadDir(dir, location)) != NULL) + /* Extract the version update graph from the script directory */ + evi_list = get_ext_ver_list(pcontrol); + + /* For each installable version ... */ + foreach(lc, evi_list) { + ExtensionVersionInfo *evi = (ExtensionVersionInfo *) lfirst(lc); ExtensionControlFile *control; - char *vername; Datum values[7]; bool nulls[7]; + ListCell *lc2; - /* must be a .sql file ... */ - if (!is_extension_script_filename(de->d_name)) - continue; - - /* ... matching extension name followed by separator */ - if (strncmp(de->d_name, pcontrol->name, extnamelen) != 0 || - de->d_name[extnamelen] != '-' || - de->d_name[extnamelen + 1] != '-') - continue; - - /* extract version name from 'extname--something.sql' filename */ - vername = pstrdup(de->d_name + extnamelen + 2); - *strrchr(vername, '.') = '\0'; - - /* ignore it if it's an update script */ - if (strstr(vername, "--")) + if (!evi->installable) continue; /* * Fetch parameters for specific version (pcontrol is not changed) */ - control = read_extension_aux_control_file(pcontrol, vername); + control = read_extension_aux_control_file(pcontrol, evi->name); memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); @@ -1959,7 +2100,7 @@ get_available_versions_for_extension(ExtensionControlFile *pcontrol, values[0] = DirectFunctionCall1(namein, CStringGetDatum(control->name)); /* version */ - values[1] = CStringGetTextDatum(vername); + values[1] = CStringGetTextDatum(evi->name); /* superuser */ values[2] = BoolGetDatum(control->superuser); /* relocatable */ @@ -1974,27 +2115,7 @@ get_available_versions_for_extension(ExtensionControlFile *pcontrol, if (control->requires == NIL) nulls[5] = true; else - { - Datum *datums; - int ndatums; - ArrayType *a; - ListCell *lc; - - ndatums = list_length(control->requires); - datums = (Datum *) palloc(ndatums * sizeof(Datum)); - ndatums = 0; - foreach(lc, control->requires) - { - char *curreq = (char *) lfirst(lc); - - datums[ndatums++] = - DirectFunctionCall1(namein, CStringGetDatum(curreq)); - } - a = construct_array(datums, ndatums, - NAMEOID, - NAMEDATALEN, false, 'c'); - values[5] = PointerGetDatum(a); - } + values[5] = convert_requires_to_datum(control->requires); /* comment */ if (control->comment == NULL) nulls[6] = true; @@ -2002,9 +2123,75 @@ get_available_versions_for_extension(ExtensionControlFile *pcontrol, values[6] = CStringGetTextDatum(control->comment); tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + /* + * Find all non-directly-installable versions that would be installed + * starting from this version, and report them, inheriting the + * parameters that aren't changed in updates from this version. + */ + foreach(lc2, evi_list) + { + ExtensionVersionInfo *evi2 = (ExtensionVersionInfo *) lfirst(lc2); + List *best_path; + + if (evi2->installable) + continue; + if (find_install_path(evi_list, evi2, &best_path) == evi) + { + /* + * Fetch parameters for this version (pcontrol is not changed) + */ + control = read_extension_aux_control_file(pcontrol, evi2->name); + + /* name stays the same */ + /* version */ + values[1] = CStringGetTextDatum(evi2->name); + /* superuser */ + values[2] = BoolGetDatum(control->superuser); + /* relocatable */ + values[3] = BoolGetDatum(control->relocatable); + /* schema stays the same */ + /* requires */ + if (control->requires == NIL) + nulls[5] = true; + else + { + values[5] = convert_requires_to_datum(control->requires); + nulls[5] = false; + } + /* comment stays the same */ + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + } } +} - FreeDir(dir); +/* + * Convert a list of extension names to a name[] Datum + */ +static Datum +convert_requires_to_datum(List *requires) +{ + Datum *datums; + int ndatums; + ArrayType *a; + ListCell *lc; + + ndatums = list_length(requires); + datums = (Datum *) palloc(ndatums * sizeof(Datum)); + ndatums = 0; + foreach(lc, requires) + { + char *curreq = (char *) lfirst(lc); + + datums[ndatums++] = + DirectFunctionCall1(namein, CStringGetDatum(curreq)); + } + a = construct_array(datums, ndatums, + NAMEOID, + NAMEDATALEN, false, 'c'); + return PointerGetDatum(a); } /* @@ -2076,7 +2263,7 @@ pg_extension_update_paths(PG_FUNCTION_ARGS) continue; /* Find shortest path from evi1 to evi2 */ - path = find_update_path(evi_list, evi1, evi2, true); + path = find_update_path(evi_list, evi1, evi2, false, true); /* Emit result row */ memset(values, 0, sizeof(values)); @@ -2808,7 +2995,8 @@ ExecAlterExtensionStmt(ParseState *pstate, AlterExtensionStmt *stmt) * time */ ApplyExtensionUpdates(extensionOid, control, - oldVersionName, updateVersions); + oldVersionName, updateVersions, + NULL, false, false); ObjectAddressSet(address, ExtensionRelationId, extensionOid); @@ -2827,7 +3015,10 @@ static void ApplyExtensionUpdates(Oid extensionOid, ExtensionControlFile *pcontrol, const char *initialVersion, - List *updateVersions) + List *updateVersions, + char *origSchemaName, + bool cascade, + bool is_create) { const char *oldVersionName = initialVersion; ListCell *lcv; @@ -2906,8 +3097,9 @@ ApplyExtensionUpdates(Oid extensionOid, heap_close(extRel, RowExclusiveLock); /* - * Look up the prerequisite extensions for this version, and build - * lists of their OIDs and the OIDs of their target schemas. + * Look up the prerequisite extensions for this version, install them + * if necessary, and build lists of their OIDs and the OIDs of their + * target schemas. */ requiredExtensions = NIL; requiredSchemas = NIL; @@ -2917,16 +3109,12 @@ ApplyExtensionUpdates(Oid extensionOid, Oid reqext; Oid reqschema; - /* - * We intentionally don't use get_extension_oid's default error - * message here, because it would be confusing in this context. - */ - reqext = get_extension_oid(curreq, true); - if (!OidIsValid(reqext)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("required extension \"%s\" is not installed", - curreq))); + reqext = get_required_extension(curreq, + control->name, + origSchemaName, + cascade, + NIL, + is_create); reqschema = get_extension_schema(reqext); requiredExtensions = lappend_oid(requiredExtensions, reqext); requiredSchemas = lappend_oid(requiredSchemas, reqschema); From 52803098ab26051c0c9802f3c19edffc90de8843 Mon Sep 17 00:00:00 2001 From: Kevin Grittner Date: Sun, 11 Sep 2016 15:37:27 -0500 Subject: [PATCH 155/871] psql tab completion for CREATE DATABASE ... TEMPLATE ... Sehrope Sarkuni, reviewed by Merlin Moncure & Vitaly Burovoy with some editing by me --- src/bin/psql/tab-complete.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 019f75a376..3e2f0846e7 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -599,8 +599,11 @@ static const SchemaQuery Query_for_list_of_matviews = { " OR '\"' || nspname || '\"' ='%s') " #define Query_for_list_of_template_databases \ -"SELECT pg_catalog.quote_ident(datname) FROM pg_catalog.pg_database "\ -" WHERE substring(pg_catalog.quote_ident(datname),1,%d)='%s' AND datistemplate" +"SELECT pg_catalog.quote_ident(d.datname) "\ +" FROM pg_catalog.pg_database d "\ +" JOIN pg_catalog.pg_roles r ON r.rolname = CURRENT_USER "\ +" WHERE substring(pg_catalog.quote_ident(d.datname),1,%d)='%s' "\ +" AND (d.datistemplate OR r.rolsuper OR d.datdba = r.oid)" #define Query_for_list_of_databases \ "SELECT pg_catalog.quote_ident(datname) FROM pg_catalog.pg_database "\ From c3c0d7bd701dae4737c974a59ffa9b366110f9c1 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Sun, 11 Sep 2016 23:26:18 +0100 Subject: [PATCH 156/871] Raise max setting of checkpoint_timeout to 1d Previously checkpoint_timeout was capped at 3600s New max setting is 86400s = 24h = 1d Discussion: 32558.1454471895@sss.pgh.pa.us --- doc/src/sgml/config.sgml | 2 +- src/backend/utils/misc/guc.c | 2 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7c483c6ef3..cd66abc8ba 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2614,7 +2614,7 @@ include_dir 'conf.d' Maximum time between automatic WAL checkpoints, in seconds. - The valid range is between 30 seconds and one hour. + The valid range is between 30 seconds and one day. The default is five minutes (5min). Increasing this parameter can increase the amount of time needed for crash recovery. diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index c5178f7cad..c72bd6190a 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2250,7 +2250,7 @@ static struct config_int ConfigureNamesInt[] = GUC_UNIT_S }, &CheckPointTimeout, - 300, 30, 3600, + 300, 30, 86400, NULL, NULL, NULL }, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 6d0666c44f..b1c3aea9ee 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -203,7 +203,7 @@ # - Checkpoints - -#checkpoint_timeout = 5min # range 30s-1h +#checkpoint_timeout = 5min # range 30s-1d #max_wal_size = 1GB #min_wal_size = 80MB #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 From fc3d4a44e9375f79709f470cb3c83d4ca28fb370 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Mon, 12 Sep 2016 08:57:14 +0100 Subject: [PATCH 157/871] Identify walsenders in pg_stat_activity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following 8299471c37fff0b walsender procs are now visible in pg_stat_activity. Set query to ‘walsender’ for walsender procs to allow them to be identified. Discussion:CAB7nPqS8c76KPSufK_HSDeYrbtg+zZ7D0EEkjeM6txSEuCB_jA@mail.gmail.com Michael Paquier, issue raised by Fujii Masao, reviewed by Tom Lane --- src/backend/replication/walsender.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 1ea2a5cfdf..c7743da034 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1806,6 +1806,9 @@ WalSndLoop(WalSndSendDataCallback send_data) last_reply_timestamp = GetCurrentTimestamp(); waiting_for_ping_response = false; + /* Report to pgstat that this process is a WAL sender */ + pgstat_report_activity(STATE_RUNNING, "walsender"); + /* * Loop until we reach the end of this timeline or the client requests to * stop streaming. From 4068eb9918cbbeaba8042fa6fe0c1f5382f2f05f Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Mon, 12 Sep 2016 09:01:58 +0100 Subject: [PATCH 158/871] Fix copy/pasto in file identification Daniel Gustafsson --- src/backend/storage/ipc/dsm_impl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/ipc/dsm_impl.c b/src/backend/storage/ipc/dsm_impl.c index c07a5c6b15..99051f02c8 100644 --- a/src/backend/storage/ipc/dsm_impl.c +++ b/src/backend/storage/ipc/dsm_impl.c @@ -41,7 +41,7 @@ * * * IDENTIFICATION - * src/backend/storage/ipc/dsm.c + * src/backend/storage/ipc/dsm_impl.c * *------------------------------------------------------------------------- */ From 63c1a871940c7c4798788e98fdb1a24408a49d05 Mon Sep 17 00:00:00 2001 From: Kevin Grittner Date: Mon, 12 Sep 2016 09:22:57 -0500 Subject: [PATCH 159/871] Fix recent commit for tab-completion of database template. The details of commit 52803098ab26051c0c9802f3c19edffc90de8843 were based on a misunderstanding of the role inheritance allowing use of a database for a template. While the CREATEDB privilege is not inherited, the database ownership is privileges are. Pointed out by Vitaly Burovoy and Tom Lane. Fix provided by Tom Lane, reviewed by Vitaly Burovoy. --- src/bin/psql/tab-complete.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 3e2f0846e7..50a45eb928 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -601,9 +601,8 @@ static const SchemaQuery Query_for_list_of_matviews = { #define Query_for_list_of_template_databases \ "SELECT pg_catalog.quote_ident(d.datname) "\ " FROM pg_catalog.pg_database d "\ -" JOIN pg_catalog.pg_roles r ON r.rolname = CURRENT_USER "\ " WHERE substring(pg_catalog.quote_ident(d.datname),1,%d)='%s' "\ -" AND (d.datistemplate OR r.rolsuper OR d.datdba = r.oid)" +" AND (d.datistemplate OR pg_catalog.pg_has_role(d.datdba, 'USAGE'))" #define Query_for_list_of_databases \ "SELECT pg_catalog.quote_ident(datname) FROM pg_catalog.pg_database "\ From 9083353b15c3cf8e7bbac104a81ad42281178cdf Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 12 Sep 2016 12:00:00 -0400 Subject: [PATCH 160/871] pg_basebackup: Clean created directories on failure Like initdb, clean up created data and xlog directories, unless the new -n/--noclean option is specified. Tablespace directories are not cleaned up, but a message is written about that. Reviewed-by: Masahiko Sawada --- doc/src/sgml/ref/pg_basebackup.sgml | 18 ++++ src/bin/pg_basebackup/pg_basebackup.c | 98 ++++++++++++++++++-- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 10 +- 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 03615da480..9f1eae12d8 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -398,6 +398,24 @@ PostgreSQL documentation + + + + + + By default, when pg_basebackup aborts with an + error, it removes any directories it might have created before + discovering that it cannot finish the job (for example, data directory + and transaction log directory). This option inhibits tidying-up and is + thus useful for debugging. + + + + Note that tablespace directories are not cleaned up either way. + + + + diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 351a42068f..42f3b273a6 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -58,6 +58,7 @@ static TablespaceList tablespace_dirs = {NULL, NULL}; static char *xlog_dir = ""; static char format = 'p'; /* p(lain)/t(ar) */ static char *label = "pg_basebackup base backup"; +static bool noclean = false; static bool showprogress = false; static int verbose = 0; static int compresslevel = 0; @@ -69,6 +70,13 @@ static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static pg_time_t last_progress_report = 0; static int32 maxrate = 0; /* no limit by default */ +static bool success = false; +static bool made_new_pgdata = false; +static bool found_existing_pgdata = false; +static bool made_new_xlogdir = false; +static bool found_existing_xlogdir = false; +static bool made_tablespace_dirs = false; +static bool found_tablespace_dirs = false; /* Progress counters */ static uint64 totalsize; @@ -82,6 +90,7 @@ static int bgpipe[2] = {-1, -1}; /* Handle to child process */ static pid_t bgchild = -1; +static bool in_log_streamer = false; /* End position for xlog streaming, empty string if unknown yet */ static XLogRecPtr xlogendptr; @@ -98,7 +107,7 @@ static PQExpBuffer recoveryconfcontents = NULL; /* Function headers */ static void usage(void); static void disconnect_and_exit(int code); -static void verify_dir_is_empty_or_create(char *dirname); +static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found); static void progress_report(int tablespacenum, const char *filename, bool force); static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum); @@ -114,6 +123,69 @@ static const char *get_tablespace_mapping(const char *dir); static void tablespace_list_append(const char *arg); +static void +cleanup_directories_atexit(void) +{ + if (success || in_log_streamer) + return; + + if (!noclean) + { + if (made_new_pgdata) + { + fprintf(stderr, _("%s: removing data directory \"%s\"\n"), + progname, basedir); + if (!rmtree(basedir, true)) + fprintf(stderr, _("%s: failed to remove data directory\n"), + progname); + } + else if (found_existing_pgdata) + { + fprintf(stderr, + _("%s: removing contents of data directory \"%s\"\n"), + progname, basedir); + if (!rmtree(basedir, false)) + fprintf(stderr, _("%s: failed to remove contents of data directory\n"), + progname); + } + + if (made_new_xlogdir) + { + fprintf(stderr, _("%s: removing transaction log directory \"%s\"\n"), + progname, xlog_dir); + if (!rmtree(xlog_dir, true)) + fprintf(stderr, _("%s: failed to remove transaction log directory\n"), + progname); + } + else if (found_existing_xlogdir) + { + fprintf(stderr, + _("%s: removing contents of transaction log directory \"%s\"\n"), + progname, xlog_dir); + if (!rmtree(xlog_dir, false)) + fprintf(stderr, _("%s: failed to remove contents of transaction log directory\n"), + progname); + } + } + else + { + if (made_new_pgdata || found_existing_pgdata) + fprintf(stderr, + _("%s: data directory \"%s\" not removed at user's request\n"), + progname, basedir); + + if (made_new_xlogdir || found_existing_xlogdir) + fprintf(stderr, + _("%s: transaction log directory \"%s\" not removed at user's request\n"), + progname, xlog_dir); + } + + if (made_tablespace_dirs || found_tablespace_dirs) + fprintf(stderr, + _("%s: changes to tablespace directories will not be undone"), + progname); +} + static void disconnect_and_exit(int code) { @@ -253,6 +325,7 @@ usage(void) printf(_(" -c, --checkpoint=fast|spread\n" " set fast or spread checkpointing\n")); printf(_(" -l, --label=LABEL set backup label\n")); + printf(_(" -n, --noclean do not clean up after errors\n")); printf(_(" -P, --progress show progress information\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -375,6 +448,8 @@ LogStreamerMain(logstreamer_param *param) { StreamCtl stream; + in_log_streamer = true; + MemSet(&stream, 0, sizeof(stream)); stream.startpos = param->startptr; stream.timeline = param->timeline; @@ -501,7 +576,7 @@ StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier) * be give and the process ended. */ static void -verify_dir_is_empty_or_create(char *dirname) +verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found) { switch (pg_check_dir(dirname)) { @@ -517,12 +592,16 @@ verify_dir_is_empty_or_create(char *dirname) progname, dirname, strerror(errno)); disconnect_and_exit(1); } + if (created) + *created = true; return; case 1: /* * Exists, empty */ + if (found) + *found = true; return; case 2: case 3: @@ -1683,7 +1762,7 @@ BaseBackup(void) { char *path = (char *) get_tablespace_mapping(PQgetvalue(res, i, 1)); - verify_dir_is_empty_or_create(path); + verify_dir_is_empty_or_create(path, &made_tablespace_dirs, &found_tablespace_dirs); } } @@ -1892,6 +1971,7 @@ main(int argc, char **argv) {"gzip", no_argument, NULL, 'z'}, {"compress", required_argument, NULL, 'Z'}, {"label", required_argument, NULL, 'l'}, + {"noclean", no_argument, NULL, 'n'}, {"dbname", required_argument, NULL, 'd'}, {"host", required_argument, NULL, 'h'}, {"port", required_argument, NULL, 'p'}, @@ -1926,7 +2006,9 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:zZ:d:c:h:p:U:s:S:wWvP", + atexit(cleanup_directories_atexit); + + while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nzZ:d:c:h:p:U:s:S:wWvP", long_options, &option_index)) != -1) { switch (c) @@ -2001,6 +2083,9 @@ main(int argc, char **argv) case 'l': label = pg_strdup(optarg); break; + case 'n': + noclean = true; + break; case 'z': #ifdef HAVE_LIBZ compresslevel = Z_DEFAULT_COMPRESSION; @@ -2170,14 +2255,14 @@ main(int argc, char **argv) * unless we are writing to stdout. */ if (format == 'p' || strcmp(basedir, "-") != 0) - verify_dir_is_empty_or_create(basedir); + verify_dir_is_empty_or_create(basedir, &made_new_pgdata, &found_existing_pgdata); /* Create transaction log symlink, if required */ if (strcmp(xlog_dir, "") != 0) { char *linkloc; - verify_dir_is_empty_or_create(xlog_dir); + verify_dir_is_empty_or_create(xlog_dir, &made_new_xlogdir, &found_existing_xlogdir); /* form name of the place where the symlink must go */ linkloc = psprintf("%s/pg_xlog", basedir); @@ -2198,5 +2283,6 @@ main(int argc, char **argv) BaseBackup(); + success = true; return 0; } diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 6c33936d25..fd9857d67b 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -4,7 +4,7 @@ use Config; use PostgresNode; use TestLib; -use Test::More tests => 51; +use Test::More tests => 54; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -40,6 +40,14 @@ [ 'pg_basebackup', '-D', "$tempdir/backup" ], 'pg_basebackup fails because of WAL configuration'); +ok(! -d "$tempdir/backup", 'backup directory was cleaned up'); + +$node->command_fails( + [ 'pg_basebackup', '-D', "$tempdir/backup", '-n' ], + 'failing run with noclean option'); + +ok(-d "$tempdir/backup", 'backup directory was created and left behind'); + open CONF, ">>$pgdata/postgresql.conf"; print CONF "max_replication_slots = 10\n"; print CONF "max_wal_senders = 10\n"; From 42fd984c0b7b53d1bc961c9ed6bb84fe28eae52b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 12 Sep 2016 19:19:24 -0400 Subject: [PATCH 161/871] Docs: assorted minor cleanups. Standardize on "user_name" for a field name in related examples in ddl.sgml; before we had variously "user_name", "username", and "user". The last is flat wrong because it conflicts with a reserved word. Be consistent about entry capitalization in a table in func.sgml. Fix a typo in pgtrgm.sgml. Back-patch to 9.6 and 9.5 as relevant. Alexander Law --- doc/src/sgml/ddl.sgml | 44 ++++++++++++++++++++-------------------- doc/src/sgml/func.sgml | 2 +- doc/src/sgml/pgtrgm.sgml | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index a393813b38..f43352c2a9 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -1629,7 +1629,7 @@ CREATE POLICY account_managers ON accounts TO managers CREATE POLICY user_policy ON users - USING (user = current_user); + USING (user_name = current_user); @@ -1642,7 +1642,7 @@ CREATE POLICY user_policy ON users CREATE POLICY user_policy ON users USING (true) - WITH CHECK (user = current_user); + WITH CHECK (user_name = current_user); @@ -1662,7 +1662,7 @@ CREATE POLICY user_policy ON users -- Simple passwd-file based example CREATE TABLE passwd ( - username text UNIQUE NOT NULL, + user_name text UNIQUE NOT NULL, pwhash text, uid int PRIMARY KEY, gid int NOT NULL, @@ -1696,9 +1696,9 @@ CREATE POLICY all_view ON passwd FOR SELECT USING (true); -- Normal users can update their own records, but -- limit which shells a normal user is allowed to set CREATE POLICY user_mod ON passwd FOR UPDATE - USING (current_user = username) + USING (current_user = user_name) WITH CHECK ( - current_user = username AND + current_user = user_name AND shell IN ('/bin/bash','/bin/sh','/bin/dash','/bin/zsh','/bin/tcsh') ); @@ -1706,7 +1706,7 @@ CREATE POLICY user_mod ON passwd FOR UPDATE GRANT SELECT, INSERT, UPDATE, DELETE ON passwd TO admin; -- Users only get select access on public columns GRANT SELECT - (username, uid, gid, real_name, home_phone, extra_info, home_dir, shell) + (user_name, uid, gid, real_name, home_phone, extra_info, home_dir, shell) ON passwd TO public; -- Allow users to update certain columns GRANT UPDATE @@ -1725,11 +1725,11 @@ GRANT UPDATE postgres=> set role admin; SET postgres=> table passwd; - username | pwhash | uid | gid | real_name | home_phone | extra_info | home_dir | shell -----------+--------+-----+-----+-----------+--------------+------------+-------------+----------- - admin | xxx | 0 | 0 | Admin | 111-222-3333 | | /root | /bin/dash - bob | xxx | 1 | 1 | Bob | 123-456-7890 | | /home/bob | /bin/zsh - alice | xxx | 2 | 1 | Alice | 098-765-4321 | | /home/alice | /bin/zsh + user_name | pwhash | uid | gid | real_name | home_phone | extra_info | home_dir | shell +-----------+--------+-----+-----+-----------+--------------+------------+-------------+----------- + admin | xxx | 0 | 0 | Admin | 111-222-3333 | | /root | /bin/dash + bob | xxx | 1 | 1 | Bob | 123-456-7890 | | /home/bob | /bin/zsh + alice | xxx | 2 | 1 | Alice | 098-765-4321 | | /home/alice | /bin/zsh (3 rows) -- Test what Alice is able to do @@ -1737,26 +1737,26 @@ postgres=> set role alice; SET postgres=> table passwd; ERROR: permission denied for relation passwd -postgres=> select username,real_name,home_phone,extra_info,home_dir,shell from passwd; - username | real_name | home_phone | extra_info | home_dir | shell -----------+-----------+--------------+------------+-------------+----------- - admin | Admin | 111-222-3333 | | /root | /bin/dash - bob | Bob | 123-456-7890 | | /home/bob | /bin/zsh - alice | Alice | 098-765-4321 | | /home/alice | /bin/zsh +postgres=> select user_name,real_name,home_phone,extra_info,home_dir,shell from passwd; + user_name | real_name | home_phone | extra_info | home_dir | shell +-----------+-----------+--------------+------------+-------------+----------- + admin | Admin | 111-222-3333 | | /root | /bin/dash + bob | Bob | 123-456-7890 | | /home/bob | /bin/zsh + alice | Alice | 098-765-4321 | | /home/alice | /bin/zsh (3 rows) -postgres=> update passwd set username = 'joe'; +postgres=> update passwd set user_name = 'joe'; ERROR: permission denied for relation passwd -- Alice is allowed to change her own real_name, but no others postgres=> update passwd set real_name = 'Alice Doe'; UPDATE 1 -postgres=> update passwd set real_name = 'John Doe' where username = 'admin'; +postgres=> update passwd set real_name = 'John Doe' where user_name = 'admin'; UPDATE 0 postgres=> update passwd set shell = '/bin/xx'; ERROR: new row violates WITH CHECK OPTION for "passwd" postgres=> delete from passwd; ERROR: permission denied for relation passwd -postgres=> insert into passwd (username) values ('xxx'); +postgres=> insert into passwd (user_name) values ('xxx'); ERROR: permission denied for relation passwd -- Alice can change her own password; RLS silently prevents updating other rows postgres=> update passwd set pwhash = 'abc'; @@ -2055,7 +2055,7 @@ DROP SCHEMA myschema CASCADE; (since this is one of the ways to restrict the activities of your users to well-defined namespaces). The syntax for that is: -CREATE SCHEMA schemaname AUTHORIZATION username; +CREATE SCHEMA schema_name AUTHORIZATION user_name; You can even omit the schema name, in which case the schema name will be the same as the user name. See username.tablename. + user_name.table_name. This is how PostgreSQL will effectively behave if you create a per-user schema for every user. diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 5148095fb3..47fcb30da0 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -9523,7 +9523,7 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple ts_filter(vector tsvector, weights "char"[]) tsvector - Select only elements with given weights from vector + select only elements with given weights from vector ts_filter('fat:2,4 cat:3b rat:5A'::tsvector, '{a,b}') 'cat':3B 'rat':5A diff --git a/doc/src/sgml/pgtrgm.sgml b/doc/src/sgml/pgtrgm.sgml index d362b03cf3..775a7b8be7 100644 --- a/doc/src/sgml/pgtrgm.sgml +++ b/doc/src/sgml/pgtrgm.sgml @@ -104,7 +104,7 @@ the second string a most similar word not a most similar substring. The range of the result is zero (indicating that the two strings are completely dissimilar) to one (indicating that the first string is - identical to one of the word of the second string). + identical to one of the words of the second string). From bfe16d1a5dec0d23c917c37de646256c71e07ee8 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 3 Aug 2016 18:29:42 -0700 Subject: [PATCH 162/871] Add more tests for targetlist SRFs. We're considering changing the implementation of targetlist SRFs considerably, and a lot of the current behaviour isn't tested in our regression tests. Thus it seems useful to increase coverage to avoid accidental behaviour changes. It's quite possible that some of the plans here will require adjustments to avoid falling afoul of ordering differences (e.g. hashed group bys). The buildfarm will tell us. Reviewed-By: Tom Lane Discussion: <20160827214829.zo2dfb5jaikii5nw@alap3.anarazel.de> --- src/test/regress/expected/tsrf.out | 501 +++++++++++++++++++++++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/serial_schedule | 1 + src/test/regress/sql/tsrf.sql | 124 +++++++ 4 files changed, 627 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/expected/tsrf.out create mode 100644 src/test/regress/sql/tsrf.sql diff --git a/src/test/regress/expected/tsrf.out b/src/test/regress/expected/tsrf.out new file mode 100644 index 0000000000..983ce17c83 --- /dev/null +++ b/src/test/regress/expected/tsrf.out @@ -0,0 +1,501 @@ +-- +-- tsrf - targetlist set returning function tests +-- +-- simple srf +SELECT generate_series(1, 3); + generate_series +----------------- + 1 + 2 + 3 +(3 rows) + +-- parallel iteration +SELECT generate_series(1, 3), generate_series(3,5); + generate_series | generate_series +-----------------+----------------- + 1 | 3 + 2 | 4 + 3 | 5 +(3 rows) + +-- parallel iteration, different number of rows +SELECT generate_series(1, 2), generate_series(1,4); + generate_series | generate_series +-----------------+----------------- + 1 | 1 + 2 | 2 + 1 | 3 + 2 | 4 +(4 rows) + +-- srf, with SRF argument +SELECT generate_series(1, generate_series(1, 3)); + generate_series +----------------- + 1 + 1 + 2 + 1 + 2 + 3 +(6 rows) + +-- srf, with two SRF arguments +SELECT generate_series(generate_series(1,3), generate_series(2, 4)); +ERROR: functions and operators can take at most one set argument +CREATE TABLE few(id int, dataa text, datab text); +INSERT INTO few VALUES(1, 'a', 'foo'),(2, 'a', 'bar'),(3, 'b', 'bar'); +-- SRF output order of sorting is maintained, if SRF is not referenced +SELECT few.id, generate_series(1,3) g FROM few ORDER BY id DESC; + id | g +----+--- + 3 | 1 + 3 | 2 + 3 | 3 + 2 | 1 + 2 | 2 + 2 | 3 + 1 | 1 + 1 | 2 + 1 | 3 +(9 rows) + +-- but SRFs can be referenced in sort +SELECT few.id, generate_series(1,3) g FROM few ORDER BY id, g DESC; + id | g +----+--- + 1 | 3 + 1 | 2 + 1 | 1 + 2 | 3 + 2 | 2 + 2 | 1 + 3 | 3 + 3 | 2 + 3 | 1 +(9 rows) + +SELECT few.id, generate_series(1,3) g FROM few ORDER BY id, generate_series(1,3) DESC; + id | g +----+--- + 1 | 3 + 1 | 2 + 1 | 1 + 2 | 3 + 2 | 2 + 2 | 1 + 3 | 3 + 3 | 2 + 3 | 1 +(9 rows) + +-- it's weird to have ORDER BYs that increase the number of results +SELECT few.id FROM few ORDER BY id, generate_series(1,3) DESC; + id +---- + 1 + 1 + 1 + 2 + 2 + 2 + 3 + 3 + 3 +(9 rows) + +-- SRFs are computed after aggregation +SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few WHERE few.id = 1 GROUP BY few.dataa; + dataa | count | min | max | unnest +-------+-------+-----+-----+-------- + a | 1 | 1 | 1 | 1 + a | 1 | 1 | 1 | 1 + a | 1 | 1 | 1 | 3 +(3 rows) + +-- unless referenced in GROUP BY clause +SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few WHERE few.id = 1 GROUP BY few.dataa, unnest('{1,1,3}'::int[]); + dataa | count | min | max | unnest +-------+-------+-----+-----+-------- + a | 2 | 1 | 1 | 1 + a | 1 | 1 | 1 | 3 +(2 rows) + +SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few WHERE few.id = 1 GROUP BY few.dataa, 5; + dataa | count | min | max | unnest +-------+-------+-----+-----+-------- + a | 2 | 1 | 1 | 1 + a | 1 | 1 | 1 | 3 +(2 rows) + +-- check HAVING works when GROUP BY does [not] reference SRF output +SELECT dataa, generate_series(1,3), count(*) FROM few GROUP BY 1 HAVING count(*) > 1; + dataa | generate_series | count +-------+-----------------+------- + a | 1 | 2 + a | 2 | 2 + a | 3 | 2 +(3 rows) + +SELECT dataa, generate_series(1,3), count(*) FROM few GROUP BY 1, 2 HAVING count(*) > 1; + dataa | generate_series | count +-------+-----------------+------- + a | 1 | 2 + a | 2 | 2 + a | 3 | 2 +(3 rows) + +-- it's weird to have GROUP BYs that increase the number of results +SELECT few.dataa, count(*), min(id), max(id) FROM few GROUP BY few.dataa; + dataa | count | min | max +-------+-------+-----+----- + b | 1 | 3 | 3 + a | 2 | 1 | 2 +(2 rows) + +SELECT few.dataa, count(*), min(id), max(id) FROM few GROUP BY few.dataa, unnest('{1,1,3}'::int[]); + dataa | count | min | max +-------+-------+-----+----- + b | 2 | 3 | 3 + a | 4 | 1 | 2 + b | 1 | 3 | 3 + a | 2 | 1 | 2 +(4 rows) + +-- SRFs are not allowed in aggregate arguments +SELECT min(generate_series(1, 3)) FROM few; +ERROR: set-valued function called in context that cannot accept a set +-- SRFs are normally computed after window functions +SELECT id,lag(id) OVER(), count(*) OVER(), generate_series(1,3) FROM few; + id | lag | count | generate_series +----+-----+-------+----------------- + 1 | | 3 | 1 + 1 | | 3 | 2 + 1 | | 3 | 3 + 2 | 1 | 3 | 1 + 2 | 1 | 3 | 2 + 2 | 1 | 3 | 3 + 3 | 2 | 3 | 1 + 3 | 2 | 3 | 2 + 3 | 2 | 3 | 3 +(9 rows) + +-- unless referencing SRFs +SELECT SUM(count(*)) OVER(PARTITION BY generate_series(1,3) ORDER BY generate_series(1,3)), generate_series(1,3) g FROM few GROUP BY g; + sum | g +-----+--- + 3 | 1 + 3 | 2 + 3 | 3 +(3 rows) + +-- sorting + grouping +SELECT few.dataa, count(*), min(id), max(id), generate_series(1,3) FROM few GROUP BY few.dataa ORDER BY 5; + dataa | count | min | max | generate_series +-------+-------+-----+-----+----------------- + b | 1 | 3 | 3 | 1 + a | 2 | 1 | 2 | 1 + b | 1 | 3 | 3 | 2 + a | 2 | 1 | 2 | 2 + b | 1 | 3 | 3 | 3 + a | 2 | 1 | 2 | 3 +(6 rows) + +-- grouping sets are a bit special, they produce NULLs in columns not actually NULL +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab); + dataa | b | g | count +-------+-----+---+------- + a | bar | 1 | 1 + a | bar | 2 | 1 + a | foo | 1 | 1 + a | foo | 2 | 1 + a | | 1 | 2 + a | | 2 | 2 + b | bar | 1 | 1 + b | bar | 2 | 1 + b | | 1 | 1 + b | | 2 | 1 + | | 1 | 3 + | | 2 | 3 + | bar | 1 | 2 + | bar | 2 | 2 + | foo | 1 | 1 + | foo | 2 | 1 +(16 rows) + +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab) ORDER BY dataa; + dataa | b | g | count +-------+-----+---+------- + a | bar | 1 | 1 + a | bar | 2 | 1 + a | foo | 1 | 1 + a | foo | 2 | 1 + a | | 1 | 2 + a | | 2 | 2 + b | bar | 1 | 1 + b | bar | 2 | 1 + b | | 1 | 1 + b | | 2 | 1 + | | 1 | 3 + | | 2 | 3 + | bar | 1 | 2 + | bar | 2 | 2 + | foo | 1 | 1 + | foo | 2 | 1 +(16 rows) + +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab) ORDER BY g; + dataa | b | g | count +-------+-----+---+------- + a | bar | 1 | 1 + a | foo | 1 | 1 + a | | 1 | 2 + b | bar | 1 | 1 + b | | 1 | 1 + | | 1 | 3 + | bar | 1 | 2 + | foo | 1 | 1 + | foo | 2 | 1 + a | bar | 2 | 1 + b | | 2 | 1 + a | foo | 2 | 1 + | bar | 2 | 2 + a | | 2 | 2 + | | 2 | 3 + b | bar | 2 | 1 +(16 rows) + +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab, g); + dataa | b | g | count +-------+-----+---+------- + a | bar | 1 | 1 + a | bar | 2 | 1 + a | bar | | 2 + a | foo | 1 | 1 + a | foo | 2 | 1 + a | foo | | 2 + a | | | 4 + b | bar | 1 | 1 + b | bar | 2 | 1 + b | bar | | 2 + b | | | 2 + | | | 6 + a | | 1 | 2 + b | | 1 | 1 + | | 1 | 3 + a | | 2 | 2 + b | | 2 | 1 + | | 2 | 3 + | bar | 1 | 2 + | bar | 2 | 2 + | bar | | 4 + | foo | 1 | 1 + | foo | 2 | 1 + | foo | | 2 +(24 rows) + +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab, g) ORDER BY dataa; + dataa | b | g | count +-------+-----+---+------- + a | bar | 1 | 1 + a | bar | 2 | 1 + a | bar | | 2 + a | foo | 1 | 1 + a | foo | 2 | 1 + a | foo | | 2 + a | | | 4 + a | | 1 | 2 + a | | 2 | 2 + b | bar | 2 | 1 + b | | | 2 + b | | 1 | 1 + b | | 2 | 1 + b | bar | 1 | 1 + b | bar | | 2 + | foo | | 2 + | foo | 1 | 1 + | | 2 | 3 + | bar | 1 | 2 + | bar | 2 | 2 + | | | 6 + | foo | 2 | 1 + | bar | | 4 + | | 1 | 3 +(24 rows) + +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab, g) ORDER BY g; + dataa | b | g | count +-------+-----+---+------- + a | bar | 1 | 1 + a | foo | 1 | 1 + b | bar | 1 | 1 + a | | 1 | 2 + b | | 1 | 1 + | | 1 | 3 + | bar | 1 | 2 + | foo | 1 | 1 + | foo | 2 | 1 + | bar | 2 | 2 + a | | 2 | 2 + b | | 2 | 1 + a | bar | 2 | 1 + | | 2 | 3 + a | foo | 2 | 1 + b | bar | 2 | 1 + a | foo | | 2 + b | bar | | 2 + b | | | 2 + | | | 6 + a | | | 4 + | bar | | 4 + | foo | | 2 + a | bar | | 2 +(24 rows) + +-- data modification +CREATE TABLE fewmore AS SELECT generate_series(1,3) AS data; +INSERT INTO fewmore VALUES(generate_series(4,5)); +SELECT * FROM fewmore; + data +------ + 1 + 2 + 3 + 4 + 5 +(5 rows) + +-- nonsense that seems to be allowed +UPDATE fewmore SET data = generate_series(4,9); +-- SRFs are not allowed in RETURNING +INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3); +ERROR: set-valued function called in context that cannot accept a set +-- nor aggregate arguments +SELECT count(generate_series(1,3)) FROM few; +ERROR: set-valued function called in context that cannot accept a set +-- nor proper VALUES +VALUES(1, generate_series(1,2)); +ERROR: set-valued function called in context that cannot accept a set +-- DISTINCT ON is evaluated before tSRF evaluation if SRF is not +-- referenced either in ORDER BY or in the DISTINCT ON list. The ORDER +-- BY reference can be implicitly generated, if there's no other ORDER BY. +-- implicit reference (via implicit ORDER) to all columns +SELECT DISTINCT ON (a) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b); + a | b | g +---+---+--- + 1 | 1 | 1 + 3 | 2 | 1 + 5 | 3 | 1 +(3 rows) + +-- unreferenced in DISTINCT ON or ORDER BY +SELECT DISTINCT ON (a) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b) +ORDER BY a, b DESC; + a | b | g +---+---+--- + 1 | 4 | 1 + 1 | 4 | 2 + 1 | 4 | 3 + 3 | 2 | 1 + 3 | 2 | 2 + 3 | 2 | 3 + 5 | 3 | 1 + 5 | 3 | 2 + 5 | 3 | 3 +(9 rows) + +-- referenced in ORDER BY +SELECT DISTINCT ON (a) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b) +ORDER BY a, b DESC, g DESC; + a | b | g +---+---+--- + 1 | 4 | 3 + 3 | 2 | 3 + 5 | 3 | 3 +(3 rows) + +-- referenced in ORDER BY and DISTINCT ON +SELECT DISTINCT ON (a, b, g) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b) +ORDER BY a, b DESC, g DESC; + a | b | g +---+---+--- + 1 | 4 | 3 + 1 | 4 | 2 + 1 | 4 | 1 + 1 | 1 | 3 + 1 | 1 | 2 + 1 | 1 | 1 + 3 | 2 | 3 + 3 | 2 | 2 + 3 | 2 | 1 + 3 | 1 | 3 + 3 | 1 | 2 + 3 | 1 | 1 + 5 | 3 | 3 + 5 | 3 | 2 + 5 | 3 | 1 + 5 | 1 | 3 + 5 | 1 | 2 + 5 | 1 | 1 +(18 rows) + +-- only SRF mentioned in DISTINCT ON +SELECT DISTINCT ON (g) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b); + a | b | g +---+---+--- + 3 | 2 | 1 + 5 | 1 | 2 + 3 | 1 | 3 +(3 rows) + +-- LIMIT / OFFSET is evaluated after SRF evaluation +SELECT a, generate_series(1,2) FROM (VALUES(1),(2),(3)) r(a) LIMIT 2 OFFSET 2; + a | generate_series +---+----------------- + 2 | 1 + 2 | 2 +(2 rows) + +-- SRFs are not allowed in LIMIT. +SELECT 1 LIMIT generate_series(1,3); +ERROR: argument of LIMIT must not return a set +LINE 1: SELECT 1 LIMIT generate_series(1,3); + ^ +-- tSRF in correlated subquery, referencing table outside +SELECT (SELECT generate_series(1,3) LIMIT 1 OFFSET few.id) FROM few; + generate_series +----------------- + 2 + 3 + +(3 rows) + +-- tSRF in correlated subquery, referencing SRF outside +SELECT (SELECT generate_series(1,3) LIMIT 1 OFFSET g.i) FROM generate_series(0,3) g(i); + generate_series +----------------- + 1 + 2 + 3 + +(4 rows) + +-- Operators can return sets too +CREATE OPERATOR |@| (PROCEDURE = unnest, RIGHTARG = ANYARRAY); +SELECT |@|ARRAY[1,2,3]; + ?column? +---------- + 1 + 2 + 3 +(3 rows) + +-- Clean up +DROP TABLE few; +DROP TABLE fewmore; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 1cb5dfc336..8641769351 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -89,7 +89,7 @@ test: brin gin gist spgist privileges init_privs security_label collate matview # ---------- # Another group of parallel tests # ---------- -test: alter_generic alter_operator misc psql async dbsize misc_functions +test: alter_generic alter_operator misc psql async dbsize misc_functions tsrf # rules cannot run concurrently with any test that creates a view test: rules psql_crosstab amutils diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 8958d8cdb9..835cf3556c 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -123,6 +123,7 @@ test: psql test: async test: dbsize test: misc_functions +test: tsrf test: rules test: psql_crosstab test: select_parallel diff --git a/src/test/regress/sql/tsrf.sql b/src/test/regress/sql/tsrf.sql new file mode 100644 index 0000000000..633dfd64c9 --- /dev/null +++ b/src/test/regress/sql/tsrf.sql @@ -0,0 +1,124 @@ +-- +-- tsrf - targetlist set returning function tests +-- + +-- simple srf +SELECT generate_series(1, 3); + +-- parallel iteration +SELECT generate_series(1, 3), generate_series(3,5); + +-- parallel iteration, different number of rows +SELECT generate_series(1, 2), generate_series(1,4); + +-- srf, with SRF argument +SELECT generate_series(1, generate_series(1, 3)); + +-- srf, with two SRF arguments +SELECT generate_series(generate_series(1,3), generate_series(2, 4)); + +CREATE TABLE few(id int, dataa text, datab text); +INSERT INTO few VALUES(1, 'a', 'foo'),(2, 'a', 'bar'),(3, 'b', 'bar'); + +-- SRF output order of sorting is maintained, if SRF is not referenced +SELECT few.id, generate_series(1,3) g FROM few ORDER BY id DESC; + +-- but SRFs can be referenced in sort +SELECT few.id, generate_series(1,3) g FROM few ORDER BY id, g DESC; +SELECT few.id, generate_series(1,3) g FROM few ORDER BY id, generate_series(1,3) DESC; + +-- it's weird to have ORDER BYs that increase the number of results +SELECT few.id FROM few ORDER BY id, generate_series(1,3) DESC; + +-- SRFs are computed after aggregation +SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few WHERE few.id = 1 GROUP BY few.dataa; +-- unless referenced in GROUP BY clause +SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few WHERE few.id = 1 GROUP BY few.dataa, unnest('{1,1,3}'::int[]); +SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few WHERE few.id = 1 GROUP BY few.dataa, 5; + +-- check HAVING works when GROUP BY does [not] reference SRF output +SELECT dataa, generate_series(1,3), count(*) FROM few GROUP BY 1 HAVING count(*) > 1; +SELECT dataa, generate_series(1,3), count(*) FROM few GROUP BY 1, 2 HAVING count(*) > 1; + +-- it's weird to have GROUP BYs that increase the number of results +SELECT few.dataa, count(*), min(id), max(id) FROM few GROUP BY few.dataa; +SELECT few.dataa, count(*), min(id), max(id) FROM few GROUP BY few.dataa, unnest('{1,1,3}'::int[]); + +-- SRFs are not allowed in aggregate arguments +SELECT min(generate_series(1, 3)) FROM few; + +-- SRFs are normally computed after window functions +SELECT id,lag(id) OVER(), count(*) OVER(), generate_series(1,3) FROM few; +-- unless referencing SRFs +SELECT SUM(count(*)) OVER(PARTITION BY generate_series(1,3) ORDER BY generate_series(1,3)), generate_series(1,3) g FROM few GROUP BY g; + +-- sorting + grouping +SELECT few.dataa, count(*), min(id), max(id), generate_series(1,3) FROM few GROUP BY few.dataa ORDER BY 5; + +-- grouping sets are a bit special, they produce NULLs in columns not actually NULL +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab); +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab) ORDER BY dataa; +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab) ORDER BY g; +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab, g); +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab, g) ORDER BY dataa; +SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab, g) ORDER BY g; + +-- data modification +CREATE TABLE fewmore AS SELECT generate_series(1,3) AS data; +INSERT INTO fewmore VALUES(generate_series(4,5)); +SELECT * FROM fewmore; + +-- nonsense that seems to be allowed +UPDATE fewmore SET data = generate_series(4,9); + +-- SRFs are not allowed in RETURNING +INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3); +-- nor aggregate arguments +SELECT count(generate_series(1,3)) FROM few; +-- nor proper VALUES +VALUES(1, generate_series(1,2)); + +-- DISTINCT ON is evaluated before tSRF evaluation if SRF is not +-- referenced either in ORDER BY or in the DISTINCT ON list. The ORDER +-- BY reference can be implicitly generated, if there's no other ORDER BY. + +-- implicit reference (via implicit ORDER) to all columns +SELECT DISTINCT ON (a) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b); + +-- unreferenced in DISTINCT ON or ORDER BY +SELECT DISTINCT ON (a) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b) +ORDER BY a, b DESC; + +-- referenced in ORDER BY +SELECT DISTINCT ON (a) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b) +ORDER BY a, b DESC, g DESC; + +-- referenced in ORDER BY and DISTINCT ON +SELECT DISTINCT ON (a, b, g) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b) +ORDER BY a, b DESC, g DESC; + +-- only SRF mentioned in DISTINCT ON +SELECT DISTINCT ON (g) a, b, generate_series(1,3) g +FROM (VALUES (3, 2), (3,1), (1,1), (1,4), (5,3), (5,1)) AS t(a, b); + +-- LIMIT / OFFSET is evaluated after SRF evaluation +SELECT a, generate_series(1,2) FROM (VALUES(1),(2),(3)) r(a) LIMIT 2 OFFSET 2; +-- SRFs are not allowed in LIMIT. +SELECT 1 LIMIT generate_series(1,3); + +-- tSRF in correlated subquery, referencing table outside +SELECT (SELECT generate_series(1,3) LIMIT 1 OFFSET few.id) FROM few; +-- tSRF in correlated subquery, referencing SRF outside +SELECT (SELECT generate_series(1,3) LIMIT 1 OFFSET g.i) FROM generate_series(0,3) g(i); + +-- Operators can return sets too +CREATE OPERATOR |@| (PROCEDURE = unnest, RIGHTARG = ANYARRAY); +SELECT |@|ARRAY[1,2,3]; + +-- Clean up +DROP TABLE few; +DROP TABLE fewmore; From 9f478b4f19d8e26300ae19e42c26343f5791e32a Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Mon, 12 Sep 2016 18:15:10 -0700 Subject: [PATCH 163/871] Address portability issues in bfe16d1a5 test output. --- src/test/regress/expected/tsrf.out | 37 ++++++++++++------------------ src/test/regress/sql/tsrf.sql | 8 +++---- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/test/regress/expected/tsrf.out b/src/test/regress/expected/tsrf.out index 983ce17c83..805e8db9f6 100644 --- a/src/test/regress/expected/tsrf.out +++ b/src/test/regress/expected/tsrf.out @@ -130,39 +130,32 @@ SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few (2 rows) -- check HAVING works when GROUP BY does [not] reference SRF output -SELECT dataa, generate_series(1,3), count(*) FROM few GROUP BY 1 HAVING count(*) > 1; +SELECT dataa, generate_series(1,1), count(*) FROM few GROUP BY 1 HAVING count(*) > 1; dataa | generate_series | count -------+-----------------+------- a | 1 | 2 - a | 2 | 2 - a | 3 | 2 -(3 rows) +(1 row) -SELECT dataa, generate_series(1,3), count(*) FROM few GROUP BY 1, 2 HAVING count(*) > 1; +SELECT dataa, generate_series(1,1), count(*) FROM few GROUP BY 1, 2 HAVING count(*) > 1; dataa | generate_series | count -------+-----------------+------- a | 1 | 2 - a | 2 | 2 - a | 3 | 2 -(3 rows) +(1 row) -- it's weird to have GROUP BYs that increase the number of results -SELECT few.dataa, count(*), min(id), max(id) FROM few GROUP BY few.dataa; - dataa | count | min | max --------+-------+-----+----- - b | 1 | 3 | 3 - a | 2 | 1 | 2 +SELECT few.dataa, count(*) FROM few WHERE dataa = 'a' GROUP BY few.dataa ORDER BY 2; + dataa | count +-------+------- + a | 2 +(1 row) + +SELECT few.dataa, count(*) FROM few WHERE dataa = 'a' GROUP BY few.dataa, unnest('{1,1,3}'::int[]) ORDER BY 2; + dataa | count +-------+------- + a | 2 + a | 4 (2 rows) -SELECT few.dataa, count(*), min(id), max(id) FROM few GROUP BY few.dataa, unnest('{1,1,3}'::int[]); - dataa | count | min | max --------+-------+-----+----- - b | 2 | 3 | 3 - a | 4 | 1 | 2 - b | 1 | 3 | 3 - a | 2 | 1 | 2 -(4 rows) - -- SRFs are not allowed in aggregate arguments SELECT min(generate_series(1, 3)) FROM few; ERROR: set-valued function called in context that cannot accept a set diff --git a/src/test/regress/sql/tsrf.sql b/src/test/regress/sql/tsrf.sql index 633dfd64c9..524779581d 100644 --- a/src/test/regress/sql/tsrf.sql +++ b/src/test/regress/sql/tsrf.sql @@ -37,12 +37,12 @@ SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few SELECT few.dataa, count(*), min(id), max(id), unnest('{1,1,3}'::int[]) FROM few WHERE few.id = 1 GROUP BY few.dataa, 5; -- check HAVING works when GROUP BY does [not] reference SRF output -SELECT dataa, generate_series(1,3), count(*) FROM few GROUP BY 1 HAVING count(*) > 1; -SELECT dataa, generate_series(1,3), count(*) FROM few GROUP BY 1, 2 HAVING count(*) > 1; +SELECT dataa, generate_series(1,1), count(*) FROM few GROUP BY 1 HAVING count(*) > 1; +SELECT dataa, generate_series(1,1), count(*) FROM few GROUP BY 1, 2 HAVING count(*) > 1; -- it's weird to have GROUP BYs that increase the number of results -SELECT few.dataa, count(*), min(id), max(id) FROM few GROUP BY few.dataa; -SELECT few.dataa, count(*), min(id), max(id) FROM few GROUP BY few.dataa, unnest('{1,1,3}'::int[]); +SELECT few.dataa, count(*) FROM few WHERE dataa = 'a' GROUP BY few.dataa ORDER BY 2; +SELECT few.dataa, count(*) FROM few WHERE dataa = 'a' GROUP BY few.dataa, unnest('{1,1,3}'::int[]) ORDER BY 2; -- SRFs are not allowed in aggregate arguments SELECT min(generate_series(1, 3)) FROM few; From 0dba54f1666ead71c54ce100b39efda67596d297 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Mon, 12 Sep 2016 19:01:16 -0700 Subject: [PATCH 164/871] Remove user_relns() SRF from regression tests. The output of the function changes whenever previous (or, as in this case, concurrent) tests leave a table in place. That causes unneeded churn. This should fix failures due to the tests added bfe16d1a5, like on lapwing, caused by the tsrf test running concurrently with misc. Those could also have been addressed by using temp tables, but that test has annoyed me before. Discussion: <27626.1473729905@sss.pgh.pa.us> --- .../regress/input/create_function_2.source | 9 -- src/test/regress/input/misc.source | 3 - .../regress/output/create_function_2.source | 8 - src/test/regress/output/misc.source | 138 ------------------ 4 files changed, 158 deletions(-) diff --git a/src/test/regress/input/create_function_2.source b/src/test/regress/input/create_function_2.source index c518559777..3c26b2fec6 100644 --- a/src/test/regress/input/create_function_2.source +++ b/src/test/regress/input/create_function_2.source @@ -62,15 +62,6 @@ CREATE FUNCTION equipment_named_ambiguous_2b(hobby text) LANGUAGE SQL; -CREATE FUNCTION user_relns() - RETURNS setof name - AS 'select relname - from pg_class c, pg_namespace n - where relnamespace = n.oid and - (nspname !~ ''pg_.*'' and nspname <> ''information_schema'') and - relkind <> ''i'' ' - LANGUAGE SQL; - CREATE FUNCTION pt_in_widget(point, widget) RETURNS bool AS '@libdir@/regress@DLSUFFIX@' diff --git a/src/test/regress/input/misc.source b/src/test/regress/input/misc.source index e16dc21f40..dd2d1b2033 100644 --- a/src/test/regress/input/misc.source +++ b/src/test/regress/input/misc.source @@ -218,9 +218,6 @@ SELECT (p.hobbies).equipment.name, name(p.hobbies), p.name FROM ONLY person p; SELECT name(equipment(p.hobbies)), name(p.hobbies), p.name FROM person* p; -SELECT user_relns() AS user_relns - ORDER BY user_relns; - SELECT name(equipment(hobby_construct(text 'skywalking', text 'mer'))); SELECT name(equipment(hobby_construct_named(text 'skywalking', text 'mer'))); diff --git a/src/test/regress/output/create_function_2.source b/src/test/regress/output/create_function_2.source index 829393243e..bdd1b1bec5 100644 --- a/src/test/regress/output/create_function_2.source +++ b/src/test/regress/output/create_function_2.source @@ -47,14 +47,6 @@ CREATE FUNCTION equipment_named_ambiguous_2b(hobby text) RETURNS setof equipment_r AS 'select * from equipment_r where equipment_r.hobby = hobby' LANGUAGE SQL; -CREATE FUNCTION user_relns() - RETURNS setof name - AS 'select relname - from pg_class c, pg_namespace n - where relnamespace = n.oid and - (nspname !~ ''pg_.*'' and nspname <> ''information_schema'') and - relkind <> ''i'' ' - LANGUAGE SQL; CREATE FUNCTION pt_in_widget(point, widget) RETURNS bool AS '@libdir@/regress@DLSUFFIX@' diff --git a/src/test/regress/output/misc.source b/src/test/regress/output/misc.source index 5c88aadc5d..574ef0d2e3 100644 --- a/src/test/regress/output/misc.source +++ b/src/test/regress/output/misc.source @@ -574,144 +574,6 @@ SELECT name(equipment(p.hobbies)), name(p.hobbies), p.name FROM person* p; peet's coffee | posthacking | jeff (6 rows) -SELECT user_relns() AS user_relns - ORDER BY user_relns; - user_relns ---------------------- - a - a_star - abstime_tbl - aggtest - aggtype - array_index_op_test - array_op_test - arrtest - b - b_star - box_tbl - bprime - brinopers - brintest - bt_f8_heap - bt_i4_heap - bt_name_heap - bt_txt_heap - btree_tall_tbl - c - c_star - char_tbl - check2_tbl - check_seq - check_tbl - circle_tbl - city - copy_tbl - d - d_star - date_tbl - default_seq - default_tbl - defaultexpr_tbl - dept - dupindexcols - e_star - emp - equipment_r - f_star - fast_emp4000 - float4_tbl - float8_tbl - func_index_heap - gin_test_tbl - gist_point_tbl - hash_f8_heap - hash_i4_heap - hash_name_heap - hash_txt_heap - hobbies_r - iexit - ihighway - inet_tbl - inhf - inhx - insert_seq - insert_tbl - int2_tbl - int4_tbl - int8_tbl - interval_tbl - iportaltest - kd_point_tbl - line_tbl - log_table - lseg_tbl - main_table - money_data - mvtest_bb - mvtest_t - mvtest_tm - mvtest_tmm - mvtest_tv - mvtest_tvm - mvtest_tvmm - mvtest_tvv - mvtest_tvvm - mvtest_tvvmv - num_data - num_exp_add - num_exp_div - num_exp_ln - num_exp_log10 - num_exp_mul - num_exp_power_10_ln - num_exp_sqrt - num_exp_sub - num_input_test - num_result - onek - onek2 - path_tbl - person - point_tbl - polygon_tbl - quad_point_tbl - radix_text_tbl - ramp - random_tbl - real_city - reltime_tbl - rls_tbl - rls_tbl_force - road - shighway - slow_emp4000 - spgist_point_tbl - spgist_text_tbl - street - stud_emp - student - subselect_tbl - tenk1 - tenk2 - test_range_excl - test_range_gist - test_range_spgist - test_tablesample - test_tablesample_v1 - test_tablesample_v2 - test_tsvector - testjsonb - text_tbl - time_tbl - timestamp_tbl - timestamptz_tbl - timetz_tbl - tinterval_tbl - toyemp - varchar_tbl - xacttest -(132 rows) - SELECT name(equipment(hobby_construct(text 'skywalking', text 'mer'))); name ------ From 445a38aba26cb80a4506af2248e3b425f795a099 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 13 Sep 2016 09:21:35 -0400 Subject: [PATCH 165/871] Have heapam.h include lockdefs.h rather than lock.h. lockdefs.h was only split from lock.h relatively recently, and represents a minimal subset of the old lock.h. heapam.h only needs that smaller subset, so adjust it to include only that. This requires some corresponding adjustments elsewhere. Peter Geoghegan --- src/backend/access/heap/syncscan.c | 2 ++ src/backend/access/transam/commit_ts.c | 1 + src/include/access/heapam.h | 2 +- src/include/access/relscan.h | 1 + src/include/nodes/execnodes.h | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/backend/access/heap/syncscan.c b/src/backend/access/heap/syncscan.c index a0f500edc8..8cc052eb7d 100644 --- a/src/backend/access/heap/syncscan.c +++ b/src/backend/access/heap/syncscan.c @@ -48,6 +48,8 @@ #include "access/heapam.h" #include "miscadmin.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" #include "utils/rel.h" diff --git a/src/backend/access/transam/commit_ts.c b/src/backend/access/transam/commit_ts.c index e330105217..a8d275f4d3 100644 --- a/src/backend/access/transam/commit_ts.c +++ b/src/backend/access/transam/commit_ts.c @@ -32,6 +32,7 @@ #include "funcapi.h" #include "miscadmin.h" #include "pg_trace.h" +#include "storage/shmem.h" #include "utils/builtins.h" #include "utils/snapmgr.h" #include "utils/timestamp.h" diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index b3a595c67e..0d12bbbbea 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -19,7 +19,7 @@ #include "nodes/lockoptions.h" #include "nodes/primnodes.h" #include "storage/bufpage.h" -#include "storage/lock.h" +#include "storage/lockdefs.h" #include "utils/relcache.h" #include "utils/snapshot.h" diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index 49c2a6f2ce..de98dd6598 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -19,6 +19,7 @@ #include "access/htup_details.h" #include "access/itup.h" #include "access/tupdesc.h" +#include "storage/spin.h" /* * Shared state for parallel heap scan. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index e28477d82d..4fa366178f 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -20,6 +20,7 @@ #include "lib/pairingheap.h" #include "nodes/params.h" #include "nodes/plannodes.h" +#include "utils/hsearch.h" #include "utils/reltrigger.h" #include "utils/sortsupport.h" #include "utils/tuplestore.h" From a4c35ea1c2f05dd5b56739fcd0dc36a4870ea0c0 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 13 Sep 2016 13:54:24 -0400 Subject: [PATCH 166/871] Improve parser's and planner's handling of set-returning functions. Teach the parser to reject misplaced set-returning functions during parse analysis using p_expr_kind, in much the same way as we do for aggregates and window functions (cf commit eaccfded9). While this isn't complete (it misses nesting-based restrictions), it's much better than the previous error reporting for such cases, and it allows elimination of assorted ad-hoc expression_returns_set() error checks. We could add nesting checks later if it seems important to catch all cases at parse time. There is one case the parser will now throw error for although previous versions allowed it, which is SRFs in the tlist of an UPDATE. That never behaved sensibly (since it's ill-defined which generated row should be used to perform the update) and it's hard to see why it should not be treated as an error. It's a release-note-worthy change though. Also, add a new Query field hasTargetSRFs reporting whether there are any SRFs in the targetlist (including GROUP BY/ORDER BY expressions). The parser can now set that basically for free during parse analysis, and we can use it in a number of places to avoid expression_returns_set searches. (There will be more such checks soon.) In some places, this allows decontorting the logic since it's no longer expensive to check for SRFs in the tlist --- so I made the checks parallel to the handling of hasAggs/hasWindowFuncs wherever it seemed appropriate. catversion bump because adding a Query field changes stored rules. Andres Freund and Tom Lane Discussion: <24639.1473782855@sss.pgh.pa.us> --- src/backend/catalog/heap.c | 9 +- src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/equalfuncs.c | 1 + src/backend/nodes/outfuncs.c | 1 + src/backend/nodes/readfuncs.c | 1 + src/backend/optimizer/path/allpaths.c | 6 +- src/backend/optimizer/plan/analyzejoins.c | 7 +- src/backend/optimizer/plan/planner.c | 20 ++- src/backend/optimizer/plan/subselect.c | 10 +- src/backend/optimizer/prep/prepjointree.c | 18 +-- src/backend/optimizer/util/clauses.c | 17 +-- src/backend/parser/analyze.c | 7 +- src/backend/parser/parse_func.c | 148 ++++++++++++++++++++++ src/backend/parser/parse_oper.c | 4 + src/backend/parser/parse_utilcmd.c | 20 +-- src/backend/rewrite/rewriteHandler.c | 2 +- src/include/catalog/catversion.h | 2 +- src/include/nodes/parsenodes.h | 1 + src/include/parser/parse_func.h | 2 + src/include/parser/parse_node.h | 3 +- src/pl/plpgsql/src/pl_exec.c | 1 + src/test/regress/expected/tsrf.out | 13 +- src/test/regress/sql/tsrf.sql | 4 +- 23 files changed, 225 insertions(+), 73 deletions(-) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index e997b574ca..dbd609493f 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -2560,14 +2560,9 @@ cookDefault(ParseState *pstate, /* * transformExpr() should have already rejected subqueries, aggregates, - * and window functions, based on the EXPR_KIND_ for a default expression. - * - * It can't return a set either. + * window functions, and SRFs, based on the EXPR_KIND_ for a default + * expression. */ - if (expression_returns_set(expr)) - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("default expression must not return a set"))); /* * Coerce the expression to the correct type and typmod, if given. This diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 4f39dad66b..71714bc1d6 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2731,6 +2731,7 @@ _copyQuery(const Query *from) COPY_SCALAR_FIELD(resultRelation); COPY_SCALAR_FIELD(hasAggs); COPY_SCALAR_FIELD(hasWindowFuncs); + COPY_SCALAR_FIELD(hasTargetSRFs); COPY_SCALAR_FIELD(hasSubLinks); COPY_SCALAR_FIELD(hasDistinctOn); COPY_SCALAR_FIELD(hasRecursive); diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 4800165a91..29a090fc48 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -921,6 +921,7 @@ _equalQuery(const Query *a, const Query *b) COMPARE_SCALAR_FIELD(resultRelation); COMPARE_SCALAR_FIELD(hasAggs); COMPARE_SCALAR_FIELD(hasWindowFuncs); + COMPARE_SCALAR_FIELD(hasTargetSRFs); COMPARE_SCALAR_FIELD(hasSubLinks); COMPARE_SCALAR_FIELD(hasDistinctOn); COMPARE_SCALAR_FIELD(hasRecursive); diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 90fecb1338..7e092d700c 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2683,6 +2683,7 @@ _outQuery(StringInfo str, const Query *node) WRITE_INT_FIELD(resultRelation); WRITE_BOOL_FIELD(hasAggs); WRITE_BOOL_FIELD(hasWindowFuncs); + WRITE_BOOL_FIELD(hasTargetSRFs); WRITE_BOOL_FIELD(hasSubLinks); WRITE_BOOL_FIELD(hasDistinctOn); WRITE_BOOL_FIELD(hasRecursive); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 894a48fb4f..917e6c8a65 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -238,6 +238,7 @@ _readQuery(void) READ_INT_FIELD(resultRelation); READ_BOOL_FIELD(hasAggs); READ_BOOL_FIELD(hasWindowFuncs); + READ_BOOL_FIELD(hasTargetSRFs); READ_BOOL_FIELD(hasSubLinks); READ_BOOL_FIELD(hasDistinctOn); READ_BOOL_FIELD(hasRecursive); diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 04264b4335..99b6bc8f5a 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -2422,7 +2422,8 @@ check_output_expressions(Query *subquery, pushdown_safety_info *safetyInfo) continue; /* Functions returning sets are unsafe (point 1) */ - if (expression_returns_set((Node *) tle->expr)) + if (subquery->hasTargetSRFs && + expression_returns_set((Node *) tle->expr)) { safetyInfo->unsafeColumns[tle->resno] = true; continue; @@ -2835,7 +2836,8 @@ remove_unused_subquery_outputs(Query *subquery, RelOptInfo *rel) * If it contains a set-returning function, we can't remove it since * that could change the number of rows returned by the subquery. */ - if (expression_returns_set(texpr)) + if (subquery->hasTargetSRFs && + expression_returns_set(texpr)) continue; /* diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index e28a8dc533..74e4245122 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -650,6 +650,11 @@ rel_is_distinct_for(PlannerInfo *root, RelOptInfo *rel, List *clause_list) bool query_supports_distinctness(Query *query) { + /* we don't cope with SRFs, see comment below */ + if (query->hasTargetSRFs) + return false; + + /* check for features we can prove distinctness with */ if (query->distinctClause != NIL || query->groupClause != NIL || query->groupingSets != NIL || @@ -695,7 +700,7 @@ query_is_distinct_for(Query *query, List *colnos, List *opids) * specified columns, since those must be evaluated before de-duplication; * but it doesn't presently seem worth the complication to check that.) */ - if (expression_returns_set((Node *) query->targetList)) + if (query->hasTargetSRFs) return false; /* diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 174210be6c..f657ffc446 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -604,6 +604,10 @@ subquery_planner(PlannerGlobal *glob, Query *parse, preprocess_expression(root, (Node *) parse->targetList, EXPRKIND_TARGET); + /* Constant-folding might have removed all set-returning functions */ + if (parse->hasTargetSRFs) + parse->hasTargetSRFs = expression_returns_set((Node *) parse->targetList); + newWithCheckOptions = NIL; foreach(l, parse->withCheckOptions) { @@ -1702,16 +1706,14 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, * Figure out whether there's a hard limit on the number of rows that * query_planner's result subplan needs to return. Even if we know a * hard limit overall, it doesn't apply if the query has any - * grouping/aggregation operations. (XXX it also doesn't apply if the - * tlist contains any SRFs; but checking for that here seems more - * costly than it's worth, since root->limit_tuples is only used for - * cost estimates, and only in a small number of cases.) + * grouping/aggregation operations, or SRFs in the tlist. */ if (parse->groupClause || parse->groupingSets || parse->distinctClause || parse->hasAggs || parse->hasWindowFuncs || + parse->hasTargetSRFs || root->hasHavingQual) root->limit_tuples = -1.0; else @@ -1928,7 +1930,11 @@ grouping_planner(PlannerInfo *root, bool inheritance_update, * weird usage that it doesn't seem worth greatly complicating matters to * account for it. */ - tlist_rows = tlist_returns_set_rows(tlist); + if (parse->hasTargetSRFs) + tlist_rows = tlist_returns_set_rows(tlist); + else + tlist_rows = 1; + if (tlist_rows > 1) { foreach(lc, current_rel->pathlist) @@ -4995,7 +5001,8 @@ make_sort_input_target(PlannerInfo *root, * Check for SRF or volatile functions. Check the SRF case first * because we must know whether we have any postponed SRFs. */ - if (expression_returns_set((Node *) expr)) + if (parse->hasTargetSRFs && + expression_returns_set((Node *) expr)) { /* We'll decide below whether these are postponable */ col_is_srf[i] = true; @@ -5034,6 +5041,7 @@ make_sort_input_target(PlannerInfo *root, { /* For sortgroupref cols, just check if any contain SRFs */ if (!have_srf_sortcols && + parse->hasTargetSRFs && expression_returns_set((Node *) expr)) have_srf_sortcols = true; } diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index 6edefb1138..263ba45f9f 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -1562,7 +1562,7 @@ simplify_EXISTS_query(PlannerInfo *root, Query *query) { /* * We don't try to simplify at all if the query uses set operations, - * aggregates, grouping sets, modifying CTEs, HAVING, OFFSET, or FOR + * aggregates, grouping sets, SRFs, modifying CTEs, HAVING, OFFSET, or FOR * UPDATE/SHARE; none of these seem likely in normal usage and their * possible effects are complex. (Note: we could ignore an "OFFSET 0" * clause, but that traditionally is used as an optimization fence, so we @@ -1573,6 +1573,7 @@ simplify_EXISTS_query(PlannerInfo *root, Query *query) query->hasAggs || query->groupingSets || query->hasWindowFuncs || + query->hasTargetSRFs || query->hasModifyingCTE || query->havingQual || query->limitOffset || @@ -1613,13 +1614,6 @@ simplify_EXISTS_query(PlannerInfo *root, Query *query) query->limitCount = NULL; } - /* - * Mustn't throw away the targetlist if it contains set-returning - * functions; those could affect whether zero rows are returned! - */ - if (expression_returns_set((Node *) query->targetList)) - return false; - /* * Otherwise, we can throw away the targetlist, as well as any GROUP, * WINDOW, DISTINCT, and ORDER BY clauses; none of those clauses will diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index a334f15773..878db9b4ab 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -1188,8 +1188,8 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, parse->hasSubLinks |= subquery->hasSubLinks; /* - * subquery won't be pulled up if it hasAggs or hasWindowFuncs, so no work - * needed on those flags + * subquery won't be pulled up if it hasAggs, hasWindowFuncs, or + * hasTargetSRFs, so no work needed on those flags */ /* @@ -1419,8 +1419,8 @@ is_simple_subquery(Query *subquery, RangeTblEntry *rte, return false; /* - * Can't pull up a subquery involving grouping, aggregation, sorting, - * limiting, or WITH. (XXX WITH could possibly be allowed later) + * Can't pull up a subquery involving grouping, aggregation, SRFs, + * sorting, limiting, or WITH. (XXX WITH could possibly be allowed later) * * We also don't pull up a subquery that has explicit FOR UPDATE/SHARE * clauses, because pullup would cause the locking to occur semantically @@ -1430,6 +1430,7 @@ is_simple_subquery(Query *subquery, RangeTblEntry *rte, */ if (subquery->hasAggs || subquery->hasWindowFuncs || + subquery->hasTargetSRFs || subquery->groupClause || subquery->groupingSets || subquery->havingQual || @@ -1542,15 +1543,6 @@ is_simple_subquery(Query *subquery, RangeTblEntry *rte, } } - /* - * Don't pull up a subquery that has any set-returning functions in its - * targetlist. Otherwise we might well wind up inserting set-returning - * functions into places where they mustn't go, such as quals of higher - * queries. This also ensures deletion of an empty jointree is valid. - */ - if (expression_returns_set((Node *) subquery->targetList)) - return false; - /* * Don't pull up a subquery that has any volatile functions in its * targetlist. Otherwise we might introduce multiple evaluations of these diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index e1baf71e38..663ffe0535 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -4449,6 +4449,7 @@ inline_function(Oid funcid, Oid result_type, Oid result_collid, querytree->utilityStmt || querytree->hasAggs || querytree->hasWindowFuncs || + querytree->hasTargetSRFs || querytree->hasSubLinks || querytree->cteList || querytree->rtable || @@ -4489,17 +4490,13 @@ inline_function(Oid funcid, Oid result_type, Oid result_collid, Assert(!modifyTargetList); /* - * Additional validity checks on the expression. It mustn't return a set, - * and it mustn't be more volatile than the surrounding function (this is - * to avoid breaking hacks that involve pretending a function is immutable - * when it really ain't). If the surrounding function is declared strict, - * then the expression must contain only strict constructs and must use - * all of the function parameters (this is overkill, but an exact analysis - * is hard). + * Additional validity checks on the expression. It mustn't be more + * volatile than the surrounding function (this is to avoid breaking hacks + * that involve pretending a function is immutable when it really ain't). + * If the surrounding function is declared strict, then the expression + * must contain only strict constructs and must use all of the function + * parameters (this is overkill, but an exact analysis is hard). */ - if (expression_returns_set(newexpr)) - goto fail; - if (funcform->provolatile == PROVOLATILE_IMMUTABLE && contain_mutable_functions(newexpr)) goto fail; diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index eac86cce3e..870fae3f51 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -417,6 +417,7 @@ transformDeleteStmt(ParseState *pstate, DeleteStmt *stmt) qry->hasSubLinks = pstate->p_hasSubLinks; qry->hasWindowFuncs = pstate->p_hasWindowFuncs; + qry->hasTargetSRFs = pstate->p_hasTargetSRFs; qry->hasAggs = pstate->p_hasAggs; if (pstate->p_hasAggs) parseCheckAggregates(pstate, qry); @@ -819,6 +820,7 @@ transformInsertStmt(ParseState *pstate, InsertStmt *stmt) qry->rtable = pstate->p_rtable; qry->jointree = makeFromExpr(pstate->p_joinlist, NULL); + qry->hasTargetSRFs = pstate->p_hasTargetSRFs; qry->hasSubLinks = pstate->p_hasSubLinks; assign_query_collations(pstate, qry); @@ -1231,6 +1233,7 @@ transformSelectStmt(ParseState *pstate, SelectStmt *stmt) qry->hasSubLinks = pstate->p_hasSubLinks; qry->hasWindowFuncs = pstate->p_hasWindowFuncs; + qry->hasTargetSRFs = pstate->p_hasTargetSRFs; qry->hasAggs = pstate->p_hasAggs; if (pstate->p_hasAggs || qry->groupClause || qry->groupingSets || qry->havingQual) parseCheckAggregates(pstate, qry); @@ -1691,6 +1694,7 @@ transformSetOperationStmt(ParseState *pstate, SelectStmt *stmt) qry->hasSubLinks = pstate->p_hasSubLinks; qry->hasWindowFuncs = pstate->p_hasWindowFuncs; + qry->hasTargetSRFs = pstate->p_hasTargetSRFs; qry->hasAggs = pstate->p_hasAggs; if (pstate->p_hasAggs || qry->groupClause || qry->groupingSets || qry->havingQual) parseCheckAggregates(pstate, qry); @@ -2170,6 +2174,7 @@ transformUpdateStmt(ParseState *pstate, UpdateStmt *stmt) qry->rtable = pstate->p_rtable; qry->jointree = makeFromExpr(pstate->p_joinlist, qual); + qry->hasTargetSRFs = pstate->p_hasTargetSRFs; qry->hasSubLinks = pstate->p_hasSubLinks; assign_query_collations(pstate, qry); @@ -2565,7 +2570,7 @@ CheckSelectLocking(Query *qry, LockClauseStrength strength) translator: %s is a SQL row locking clause such as FOR UPDATE */ errmsg("%s is not allowed with window functions", LCS_asString(strength)))); - if (expression_returns_set((Node *) qry->targetList)) + if (qry->hasTargetSRFs) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*------ diff --git a/src/backend/parser/parse_func.c b/src/backend/parser/parse_func.c index 61af484fee..56c9a4293d 100644 --- a/src/backend/parser/parse_func.c +++ b/src/backend/parser/parse_func.c @@ -25,6 +25,7 @@ #include "parser/parse_agg.h" #include "parser/parse_clause.h" #include "parser/parse_coerce.h" +#include "parser/parse_expr.h" #include "parser/parse_func.h" #include "parser/parse_relation.h" #include "parser/parse_target.h" @@ -625,6 +626,10 @@ ParseFuncOrColumn(ParseState *pstate, List *funcname, List *fargs, exprLocation((Node *) llast(fargs))))); } + /* if it returns a set, check that's OK */ + if (retset) + check_srf_call_placement(pstate, location); + /* build the appropriate output structure */ if (fdresult == FUNCDETAIL_NORMAL) { @@ -2040,3 +2045,146 @@ LookupAggNameTypeNames(List *aggname, List *argtypes, bool noError) return oid; } + + +/* + * check_srf_call_placement + * Verify that a set-returning function is called in a valid place, + * and throw a nice error if not. + * + * A side-effect is to set pstate->p_hasTargetSRFs true if appropriate. + */ +void +check_srf_call_placement(ParseState *pstate, int location) +{ + const char *err; + bool errkind; + + /* + * Check to see if the set-returning function is in an invalid place + * within the query. Basically, we don't allow SRFs anywhere except in + * the targetlist (which includes GROUP BY/ORDER BY expressions), VALUES, + * and functions in FROM. + * + * For brevity we support two schemes for reporting an error here: set + * "err" to a custom message, or set "errkind" true if the error context + * is sufficiently identified by what ParseExprKindName will return, *and* + * what it will return is just a SQL keyword. (Otherwise, use a custom + * message to avoid creating translation problems.) + */ + err = NULL; + errkind = false; + switch (pstate->p_expr_kind) + { + case EXPR_KIND_NONE: + Assert(false); /* can't happen */ + break; + case EXPR_KIND_OTHER: + /* Accept SRF here; caller must throw error if wanted */ + break; + case EXPR_KIND_JOIN_ON: + case EXPR_KIND_JOIN_USING: + err = _("set-returning functions are not allowed in JOIN conditions"); + break; + case EXPR_KIND_FROM_SUBSELECT: + /* can't get here, but just in case, throw an error */ + errkind = true; + break; + case EXPR_KIND_FROM_FUNCTION: + /* okay ... but we can't check nesting here */ + break; + case EXPR_KIND_WHERE: + errkind = true; + break; + case EXPR_KIND_POLICY: + err = _("set-returning functions are not allowed in policy expressions"); + break; + case EXPR_KIND_HAVING: + errkind = true; + break; + case EXPR_KIND_FILTER: + errkind = true; + break; + case EXPR_KIND_WINDOW_PARTITION: + case EXPR_KIND_WINDOW_ORDER: + /* okay, these are effectively GROUP BY/ORDER BY */ + pstate->p_hasTargetSRFs = true; + break; + case EXPR_KIND_WINDOW_FRAME_RANGE: + case EXPR_KIND_WINDOW_FRAME_ROWS: + err = _("set-returning functions are not allowed in window definitions"); + break; + case EXPR_KIND_SELECT_TARGET: + case EXPR_KIND_INSERT_TARGET: + /* okay */ + pstate->p_hasTargetSRFs = true; + break; + case EXPR_KIND_UPDATE_SOURCE: + case EXPR_KIND_UPDATE_TARGET: + /* disallowed because it would be ambiguous what to do */ + errkind = true; + break; + case EXPR_KIND_GROUP_BY: + case EXPR_KIND_ORDER_BY: + /* okay */ + pstate->p_hasTargetSRFs = true; + break; + case EXPR_KIND_DISTINCT_ON: + /* okay */ + pstate->p_hasTargetSRFs = true; + break; + case EXPR_KIND_LIMIT: + case EXPR_KIND_OFFSET: + errkind = true; + break; + case EXPR_KIND_RETURNING: + errkind = true; + break; + case EXPR_KIND_VALUES: + /* okay */ + break; + case EXPR_KIND_CHECK_CONSTRAINT: + case EXPR_KIND_DOMAIN_CHECK: + err = _("set-returning functions are not allowed in check constraints"); + break; + case EXPR_KIND_COLUMN_DEFAULT: + case EXPR_KIND_FUNCTION_DEFAULT: + err = _("set-returning functions are not allowed in DEFAULT expressions"); + break; + case EXPR_KIND_INDEX_EXPRESSION: + err = _("set-returning functions are not allowed in index expressions"); + break; + case EXPR_KIND_INDEX_PREDICATE: + err = _("set-returning functions are not allowed in index predicates"); + break; + case EXPR_KIND_ALTER_COL_TRANSFORM: + err = _("set-returning functions are not allowed in transform expressions"); + break; + case EXPR_KIND_EXECUTE_PARAMETER: + err = _("set-returning functions are not allowed in EXECUTE parameters"); + break; + case EXPR_KIND_TRIGGER_WHEN: + err = _("set-returning functions are not allowed in trigger WHEN conditions"); + break; + + /* + * There is intentionally no default: case here, so that the + * compiler will warn if we add a new ParseExprKind without + * extending this switch. If we do see an unrecognized value at + * runtime, the behavior will be the same as for EXPR_KIND_OTHER, + * which is sane anyway. + */ + } + if (err) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg_internal("%s", err), + parser_errposition(pstate, location))); + if (errkind) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /* translator: %s is name of a SQL construct, eg GROUP BY */ + errmsg("set-returning functions are not allowed in %s", + ParseExprKindName(pstate->p_expr_kind)), + parser_errposition(pstate, location))); +} diff --git a/src/backend/parser/parse_oper.c b/src/backend/parser/parse_oper.c index e913d05a79..aecda6d933 100644 --- a/src/backend/parser/parse_oper.c +++ b/src/backend/parser/parse_oper.c @@ -839,6 +839,10 @@ make_op(ParseState *pstate, List *opname, Node *ltree, Node *rtree, result->args = args; result->location = location; + /* if it returns a set, check that's OK */ + if (result->opretset) + check_srf_call_placement(pstate, location); + ReleaseSysCache(tup); return (Expr *) result; diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 7a2950e6a9..0670bc2482 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -294,7 +294,7 @@ transformCreateStmt(CreateStmt *stmt, const char *queryString) * overridden if an inherited table has oids. */ stmt->options = lcons(makeDefElem("oids", - (Node *) makeInteger(cxt.hasoids), -1), + (Node *) makeInteger(cxt.hasoids), -1), stmt->options); } @@ -483,7 +483,7 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column) makeString(cxt->relation->relname), makeString(column->colname)); altseqstmt->options = list_make1(makeDefElem("owned_by", - (Node *) attnamelist, -1)); + (Node *) attnamelist, -1)); cxt->alist = lappend(cxt->alist, altseqstmt); @@ -2106,17 +2106,11 @@ transformIndexStmt(Oid relid, IndexStmt *stmt, const char *queryString) /* * transformExpr() should have already rejected subqueries, - * aggregates, and window functions, based on the EXPR_KIND_ for - * an index expression. + * aggregates, window functions, and SRFs, based on the EXPR_KIND_ + * for an index expression. * - * Also reject expressions returning sets; this is for consistency - * with what transformWhereClause() checks for the predicate. * DefineIndex() will make more checks. */ - if (expression_returns_set(ielem->expr)) - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("index expression cannot return a set"))); } } @@ -2594,12 +2588,6 @@ transformAlterTableStmt(Oid relid, AlterTableStmt *stmt, def->cooked_default = transformExpr(pstate, def->raw_default, EXPR_KIND_ALTER_COL_TRANSFORM); - - /* it can't return a set */ - if (expression_returns_set(def->cooked_default)) - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("transform expression must not return a set"))); } newcmds = lappend(newcmds, cmd); diff --git a/src/backend/rewrite/rewriteHandler.c b/src/backend/rewrite/rewriteHandler.c index a22a11e2c1..b828e3cb07 100644 --- a/src/backend/rewrite/rewriteHandler.c +++ b/src/backend/rewrite/rewriteHandler.c @@ -2221,7 +2221,7 @@ view_query_is_auto_updatable(Query *viewquery, bool check_cols) if (viewquery->hasWindowFuncs) return gettext_noop("Views that return window functions are not automatically updatable."); - if (expression_returns_set((Node *) viewquery->targetList)) + if (viewquery->hasTargetSRFs) return gettext_noop("Views that return set-returning functions are not automatically updatable."); /* diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index c04edadbf0..ef691c5721 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201608231 +#define CATALOG_VERSION_NO 201609131 #endif diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 8d3dcf4d4c..6de2cab6b2 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -116,6 +116,7 @@ typedef struct Query bool hasAggs; /* has aggregates in tlist or havingQual */ bool hasWindowFuncs; /* has window functions in tlist */ + bool hasTargetSRFs; /* has set-returning functions in tlist */ bool hasSubLinks; /* has subquery SubLink */ bool hasDistinctOn; /* distinctClause is from DISTINCT ON */ bool hasRecursive; /* WITH RECURSIVE was specified */ diff --git a/src/include/parser/parse_func.h b/src/include/parser/parse_func.h index 0cefdf1b60..ed16d36982 100644 --- a/src/include/parser/parse_func.h +++ b/src/include/parser/parse_func.h @@ -67,4 +67,6 @@ extern Oid LookupFuncNameTypeNames(List *funcname, List *argtypes, extern Oid LookupAggNameTypeNames(List *aggname, List *argtypes, bool noError); +extern void check_srf_call_placement(ParseState *pstate, int location); + #endif /* PARSE_FUNC_H */ diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h index e3e359c021..66335863db 100644 --- a/src/include/parser/parse_node.h +++ b/src/include/parser/parse_node.h @@ -27,7 +27,7 @@ * by extension code that might need to call transformExpr(). The core code * will not enforce any context-driven restrictions on EXPR_KIND_OTHER * expressions, so the caller would have to check for sub-selects, aggregates, - * and window functions if those need to be disallowed. + * window functions, SRFs, etc if those need to be disallowed. */ typedef enum ParseExprKind { @@ -150,6 +150,7 @@ struct ParseState Node *p_value_substitute; /* what to replace VALUE with, if any */ bool p_hasAggs; bool p_hasWindowFuncs; + bool p_hasTargetSRFs; bool p_hasSubLinks; bool p_hasModifyingCTE; bool p_is_insert; diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index 6141b7ab49..470cf935df 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -6799,6 +6799,7 @@ exec_simple_check_plan(PLpgSQL_execstate *estate, PLpgSQL_expr *expr) */ if (query->hasAggs || query->hasWindowFuncs || + query->hasTargetSRFs || query->hasSubLinks || query->hasForUpdate || query->cteList || diff --git a/src/test/regress/expected/tsrf.out b/src/test/regress/expected/tsrf.out index 805e8db9f6..622f75517a 100644 --- a/src/test/regress/expected/tsrf.out +++ b/src/test/regress/expected/tsrf.out @@ -359,15 +359,20 @@ SELECT * FROM fewmore; 5 (5 rows) --- nonsense that seems to be allowed +-- SRFs are not allowed in UPDATE (they once were, but it was nonsense) UPDATE fewmore SET data = generate_series(4,9); +ERROR: set-returning functions are not allowed in UPDATE +LINE 1: UPDATE fewmore SET data = generate_series(4,9); + ^ -- SRFs are not allowed in RETURNING INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3); -ERROR: set-valued function called in context that cannot accept a set +ERROR: set-returning functions are not allowed in RETURNING +LINE 1: INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3)... + ^ -- nor aggregate arguments SELECT count(generate_series(1,3)) FROM few; ERROR: set-valued function called in context that cannot accept a set --- nor proper VALUES +-- nor standalone VALUES (but surely this is a bug?) VALUES(1, generate_series(1,2)); ERROR: set-valued function called in context that cannot accept a set -- DISTINCT ON is evaluated before tSRF evaluation if SRF is not @@ -457,7 +462,7 @@ SELECT a, generate_series(1,2) FROM (VALUES(1),(2),(3)) r(a) LIMIT 2 OFFSET 2; -- SRFs are not allowed in LIMIT. SELECT 1 LIMIT generate_series(1,3); -ERROR: argument of LIMIT must not return a set +ERROR: set-returning functions are not allowed in LIMIT LINE 1: SELECT 1 LIMIT generate_series(1,3); ^ -- tSRF in correlated subquery, referencing table outside diff --git a/src/test/regress/sql/tsrf.sql b/src/test/regress/sql/tsrf.sql index 524779581d..c28dd017e5 100644 --- a/src/test/regress/sql/tsrf.sql +++ b/src/test/regress/sql/tsrf.sql @@ -68,14 +68,14 @@ CREATE TABLE fewmore AS SELECT generate_series(1,3) AS data; INSERT INTO fewmore VALUES(generate_series(4,5)); SELECT * FROM fewmore; --- nonsense that seems to be allowed +-- SRFs are not allowed in UPDATE (they once were, but it was nonsense) UPDATE fewmore SET data = generate_series(4,9); -- SRFs are not allowed in RETURNING INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3); -- nor aggregate arguments SELECT count(generate_series(1,3)) FROM few; --- nor proper VALUES +-- nor standalone VALUES (but surely this is a bug?) VALUES(1, generate_series(1,2)); -- DISTINCT ON is evaluated before tSRF evaluation if SRF is not From fdc79e1909dc3866a385ffb74bdd6ce6a082a300 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 13 Sep 2016 14:25:35 -0400 Subject: [PATCH 167/871] Fix executor/README to reflect disallowing SRFs in UPDATE. The parenthetical comment here is obsoleted by commit a4c35ea1c. Noted by Andres Freund. --- src/backend/executor/README | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/backend/executor/README b/src/backend/executor/README index 8afa1e3e4a..f1d1e4c76c 100644 --- a/src/backend/executor/README +++ b/src/backend/executor/README @@ -195,8 +195,7 @@ the entire row value in the join output row. We disallow set-returning functions in the targetlist of SELECT FOR UPDATE, so as to ensure that at most one tuple can be returned for any particular set of scan tuples. Otherwise we'd get duplicates due to the original -query returning the same set of scan tuples multiple times. (Note: there -is no explicit prohibition on SRFs in UPDATE, but the net effect will be -that only the first result row of an SRF counts, because all subsequent -rows will result in attempts to re-update an already updated target row. -This is historical behavior and seems not worth changing.) +query returning the same set of scan tuples multiple times. Likewise, +SRFs are disallowed in an UPDATE's targetlist. There, they would have the +effect of the same row being updated multiple times, which is not very +useful --- and updates after the first would have no effect anyway. From 55c3391d1e6a201b5b891781d21fe682a8c64fe6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 13 Sep 2016 17:17:48 -0400 Subject: [PATCH 168/871] Be pickier about converting between Name and Datum. We were misapplying NameGetDatum() to plain C strings in some places. This worked, because it was just a pointer cast anyway, but it's a type cheat in some sense. Use CStringGetDatum instead, and modify the NameGetDatum macro so it won't compile if applied to something that's not a pointer to NameData. This should result in no changes to generated code, but it is logically cleaner. Mark Dilger, tweaked a bit by me Discussion: --- src/backend/commands/dbcommands.c | 8 ++++---- src/backend/commands/proclang.c | 2 +- src/backend/commands/typecmds.c | 2 +- src/include/postgres.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index ef486593c0..0919ad8dfe 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -1260,7 +1260,7 @@ movedb(const char *dbname, const char *tblspcname) ScanKeyInit(&scankey, Anum_pg_database_datname, BTEqualStrategyNumber, F_NAMEEQ, - NameGetDatum(dbname)); + CStringGetDatum(dbname)); sysscan = systable_beginscan(pgdbrel, DatabaseNameIndexId, true, NULL, 1, &scankey); oldtuple = systable_getnext(sysscan); @@ -1486,7 +1486,7 @@ AlterDatabase(ParseState *pstate, AlterDatabaseStmt *stmt, bool isTopLevel) ScanKeyInit(&scankey, Anum_pg_database_datname, BTEqualStrategyNumber, F_NAMEEQ, - NameGetDatum(stmt->dbname)); + CStringGetDatum(stmt->dbname)); scan = systable_beginscan(rel, DatabaseNameIndexId, true, NULL, 1, &scankey); tuple = systable_getnext(scan); @@ -1603,7 +1603,7 @@ AlterDatabaseOwner(const char *dbname, Oid newOwnerId) ScanKeyInit(&scankey, Anum_pg_database_datname, BTEqualStrategyNumber, F_NAMEEQ, - NameGetDatum(dbname)); + CStringGetDatum(dbname)); scan = systable_beginscan(rel, DatabaseNameIndexId, true, NULL, 1, &scankey); tuple = systable_getnext(scan); @@ -1743,7 +1743,7 @@ get_db_info(const char *name, LOCKMODE lockmode, ScanKeyInit(&scanKey, Anum_pg_database_datname, BTEqualStrategyNumber, F_NAMEEQ, - NameGetDatum(name)); + CStringGetDatum(name)); scan = systable_beginscan(relation, DatabaseNameIndexId, true, NULL, 1, &scanKey); diff --git a/src/backend/commands/proclang.c b/src/backend/commands/proclang.c index 761d08f604..4f870e8d30 100644 --- a/src/backend/commands/proclang.c +++ b/src/backend/commands/proclang.c @@ -463,7 +463,7 @@ find_language_template(const char *languageName) ScanKeyInit(&key, Anum_pg_pltemplate_tmplname, BTEqualStrategyNumber, F_NAMEEQ, - NameGetDatum(languageName)); + CStringGetDatum(languageName)); scan = systable_beginscan(rel, PLTemplateNameIndexId, true, NULL, 1, &key); diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c index 41fd2dae7f..056933a584 100644 --- a/src/backend/commands/typecmds.c +++ b/src/backend/commands/typecmds.c @@ -3503,7 +3503,7 @@ AlterTypeNamespaceInternal(Oid typeOid, Oid nspOid, /* check for duplicate name (more friendly than unique-index failure) */ if (SearchSysCacheExists2(TYPENAMENSP, - CStringGetDatum(NameStr(typform->typname)), + NameGetDatum(&typform->typname), ObjectIdGetDatum(nspOid))) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), diff --git a/src/include/postgres.h b/src/include/postgres.h index d999013238..be4d0d609e 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -600,7 +600,7 @@ typedef Datum *DatumPtr; * value has adequate lifetime. */ -#define NameGetDatum(X) PointerGetDatum(X) +#define NameGetDatum(X) CStringGetDatum(NameStr(*(X))) /* * DatumGetInt64 From a163c006ca3e6026546ee0f6c487a0dcfc66f82b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 14 Sep 2016 14:30:40 -0400 Subject: [PATCH 169/871] Tweak targetlist-SRF tests. Add a test case showing that we don't support SRFs in window-function arguments. Remove a duplicate test case for SRFs in aggregate arguments. --- src/test/regress/expected/tsrf.out | 6 +++--- src/test/regress/sql/tsrf.sql | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/test/regress/expected/tsrf.out b/src/test/regress/expected/tsrf.out index 622f75517a..e9bea411fd 100644 --- a/src/test/regress/expected/tsrf.out +++ b/src/test/regress/expected/tsrf.out @@ -159,6 +159,9 @@ SELECT few.dataa, count(*) FROM few WHERE dataa = 'a' GROUP BY few.dataa, unnest -- SRFs are not allowed in aggregate arguments SELECT min(generate_series(1, 3)) FROM few; ERROR: set-valued function called in context that cannot accept a set +-- SRFs are not allowed in window function arguments, either +SELECT min(generate_series(1, 3)) OVER() FROM few; +ERROR: set-valued function called in context that cannot accept a set -- SRFs are normally computed after window functions SELECT id,lag(id) OVER(), count(*) OVER(), generate_series(1,3) FROM few; id | lag | count | generate_series @@ -369,9 +372,6 @@ INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3); ERROR: set-returning functions are not allowed in RETURNING LINE 1: INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3)... ^ --- nor aggregate arguments -SELECT count(generate_series(1,3)) FROM few; -ERROR: set-valued function called in context that cannot accept a set -- nor standalone VALUES (but surely this is a bug?) VALUES(1, generate_series(1,2)); ERROR: set-valued function called in context that cannot accept a set diff --git a/src/test/regress/sql/tsrf.sql b/src/test/regress/sql/tsrf.sql index c28dd017e5..b0dfc193ac 100644 --- a/src/test/regress/sql/tsrf.sql +++ b/src/test/regress/sql/tsrf.sql @@ -47,6 +47,9 @@ SELECT few.dataa, count(*) FROM few WHERE dataa = 'a' GROUP BY few.dataa, unnest -- SRFs are not allowed in aggregate arguments SELECT min(generate_series(1, 3)) FROM few; +-- SRFs are not allowed in window function arguments, either +SELECT min(generate_series(1, 3)) OVER() FROM few; + -- SRFs are normally computed after window functions SELECT id,lag(id) OVER(), count(*) OVER(), generate_series(1,3) FROM few; -- unless referencing SRFs @@ -73,8 +76,7 @@ UPDATE fewmore SET data = generate_series(4,9); -- SRFs are not allowed in RETURNING INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3); --- nor aggregate arguments -SELECT count(generate_series(1,3)) FROM few; + -- nor standalone VALUES (but surely this is a bug?) VALUES(1, generate_series(1,2)); From 6415ba502bdc540e21f122d4c6c87d4a35f8ec27 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 14 Sep 2016 15:43:26 -0400 Subject: [PATCH 170/871] Improve code comment for GatherPath's single_copy flag. Discussion: 5934.1472642782@sss.pgh.pa.us --- src/include/nodes/relation.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 2709cc7df5..3a1255a657 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -1190,7 +1190,7 @@ typedef struct GatherPath { Path path; Path *subpath; /* path for each worker */ - bool single_copy; /* path must not be executed >1x */ + bool single_copy; /* don't execute path more than once */ } GatherPath; /* From 0dac5b5174bde3d6fb4b444a2aa4ca1f0091e258 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 14 Sep 2016 19:48:42 -0400 Subject: [PATCH 171/871] Tweak targetlist-SRF tests some more. Seems like it would be good to have a test case documenting the existing behavior for non-top-level SRFs. --- src/test/regress/expected/tsrf.out | 11 +++++++++++ src/test/regress/sql/tsrf.sql | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/src/test/regress/expected/tsrf.out b/src/test/regress/expected/tsrf.out index e9bea411fd..d9a5f137dc 100644 --- a/src/test/regress/expected/tsrf.out +++ b/src/test/regress/expected/tsrf.out @@ -375,6 +375,17 @@ LINE 1: INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3)... -- nor standalone VALUES (but surely this is a bug?) VALUES(1, generate_series(1,2)); ERROR: set-valued function called in context that cannot accept a set +-- We allow tSRFs that are not at top level +SELECT int4mul(generate_series(1,2), 10); + int4mul +--------- + 10 + 20 +(2 rows) + +-- but SRFs in function RTEs must be at top level (annoying restriction) +SELECT * FROM int4mul(generate_series(1,2), 10); +ERROR: set-valued function called in context that cannot accept a set -- DISTINCT ON is evaluated before tSRF evaluation if SRF is not -- referenced either in ORDER BY or in the DISTINCT ON list. The ORDER -- BY reference can be implicitly generated, if there's no other ORDER BY. diff --git a/src/test/regress/sql/tsrf.sql b/src/test/regress/sql/tsrf.sql index b0dfc193ac..4f854c8b83 100644 --- a/src/test/regress/sql/tsrf.sql +++ b/src/test/regress/sql/tsrf.sql @@ -80,6 +80,12 @@ INSERT INTO fewmore VALUES(1) RETURNING generate_series(1,3); -- nor standalone VALUES (but surely this is a bug?) VALUES(1, generate_series(1,2)); +-- We allow tSRFs that are not at top level +SELECT int4mul(generate_series(1,2), 10); + +-- but SRFs in function RTEs must be at top level (annoying restriction) +SELECT * FROM int4mul(generate_series(1,2), 10); + -- DISTINCT ON is evaluated before tSRF evaluation if SRF is not -- referenced either in ORDER BY or in the DISTINCT ON list. The ORDER -- BY reference can be implicitly generated, if there's no other ORDER BY. From 656df624c0d7b50e1714f2a3a14e143e63799a80 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 8 Sep 2016 12:00:00 -0400 Subject: [PATCH 172/871] Add overflow checks to money type input function The money type input function did not have any overflow checks at all. There were some regression tests that purported to check for overflow, but they actually checked for the overflow behavior of the int8 type before casting to money. Remove those unnecessary checks and add some that actually check the money input function. Reviewed-by: Fabien COELHO --- src/backend/utils/adt/cash.c | 53 ++++++++++++++-- src/test/regress/expected/money.out | 98 ++++++++++++++++++++++++++--- src/test/regress/sql/money.sql | 30 +++++++-- 3 files changed, 165 insertions(+), 16 deletions(-) diff --git a/src/backend/utils/adt/cash.c b/src/backend/utils/adt/cash.c index b336185df7..a146b0a0bc 100644 --- a/src/backend/utils/adt/cash.c +++ b/src/backend/utils/adt/cash.c @@ -189,13 +189,30 @@ cash_in(PG_FUNCTION_ARGS) printf("cashin- string is '%s'\n", s); #endif + /* + * We accumulate the absolute amount in "value" and then apply the sign at + * the end. (The sign can appear before or after the digits, so it would + * be more complicated to do otherwise.) Because of the larger range of + * negative signed integers, we build "value" in the negative and then + * flip the sign at the end, catching most-negative-number overflow if + * necessary. + */ + for (; *s; s++) { /* we look for digits as long as we have found less */ /* than the required number of decimal places */ if (isdigit((unsigned char) *s) && (!seen_dot || dec < fpoint)) { - value = (value * 10) + (*s - '0'); + Cash newvalue = (value * 10) - (*s - '0'); + + if (newvalue / 10 != value) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("value \"%s\" is out of range for type money", + str))); + + value = newvalue; if (seen_dot) dec++; @@ -214,11 +231,27 @@ cash_in(PG_FUNCTION_ARGS) /* round off if there's another digit */ if (isdigit((unsigned char) *s) && *s >= '5') - value++; + value--; /* remember we build the value in the negative */ + + if (value > 0) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("value \"%s\" is out of range for type money", + str))); /* adjust for less than required decimal places */ for (; dec < fpoint; dec++) - value *= 10; + { + Cash newvalue = value * 10; + + if (newvalue / 10 != value) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("value \"%s\" is out of range for type money", + str))); + + value = newvalue; + } /* * should only be trailing digits followed by whitespace, right paren, @@ -247,7 +280,19 @@ cash_in(PG_FUNCTION_ARGS) str))); } - result = value * sgn; + /* If the value is supposed to be positive, flip the sign, but check for + * the most negative number. */ + if (sgn > 0) + { + result = -value; + if (result < 0) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("value \"%s\" is out of range for type money", + str))); + } + else + result = value; #ifdef CASHDEBUG printf("cashin- result is " INT64_FORMAT "\n", result); diff --git a/src/test/regress/expected/money.out b/src/test/regress/expected/money.out index 538235c4cc..5695f87500 100644 --- a/src/test/regress/expected/money.out +++ b/src/test/regress/expected/money.out @@ -185,6 +185,96 @@ SELECT * FROM money_data; $123.46 (1 row) +-- input checks +SELECT '1234567890'::money; + money +------------------- + $1,234,567,890.00 +(1 row) + +SELECT '12345678901234567'::money; + money +---------------------------- + $12,345,678,901,234,567.00 +(1 row) + +SELECT '123456789012345678'::money; +ERROR: value "123456789012345678" is out of range for type money +LINE 1: SELECT '123456789012345678'::money; + ^ +SELECT '9223372036854775807'::money; +ERROR: value "9223372036854775807" is out of range for type money +LINE 1: SELECT '9223372036854775807'::money; + ^ +SELECT '-12345'::money; + money +------------- + -$12,345.00 +(1 row) + +SELECT '-1234567890'::money; + money +-------------------- + -$1,234,567,890.00 +(1 row) + +SELECT '-12345678901234567'::money; + money +----------------------------- + -$12,345,678,901,234,567.00 +(1 row) + +SELECT '-123456789012345678'::money; +ERROR: value "-123456789012345678" is out of range for type money +LINE 1: SELECT '-123456789012345678'::money; + ^ +SELECT '-9223372036854775808'::money; +ERROR: value "-9223372036854775808" is out of range for type money +LINE 1: SELECT '-9223372036854775808'::money; + ^ +-- special characters +SELECT '(1)'::money; + money +-------- + -$1.00 +(1 row) + +SELECT '($123,456.78)'::money; + money +-------------- + -$123,456.78 +(1 row) + +-- documented minimums and maximums +SELECT '-92233720368547758.08'::money; + money +----------------------------- + -$92,233,720,368,547,758.08 +(1 row) + +SELECT '92233720368547758.07'::money; + money +---------------------------- + $92,233,720,368,547,758.07 +(1 row) + +SELECT '-92233720368547758.09'::money; +ERROR: value "-92233720368547758.09" is out of range for type money +LINE 1: SELECT '-92233720368547758.09'::money; + ^ +SELECT '92233720368547758.08'::money; +ERROR: value "92233720368547758.08" is out of range for type money +LINE 1: SELECT '92233720368547758.08'::money; + ^ +-- rounding +SELECT '-92233720368547758.085'::money; +ERROR: value "-92233720368547758.085" is out of range for type money +LINE 1: SELECT '-92233720368547758.085'::money; + ^ +SELECT '92233720368547758.075'::money; +ERROR: value "92233720368547758.075" is out of range for type money +LINE 1: SELECT '92233720368547758.075'::money; + ^ -- Cast int4/int8 to money SELECT 1234567890::money; money @@ -198,10 +288,6 @@ SELECT 12345678901234567::money; $12,345,678,901,234,567.00 (1 row) -SELECT 123456789012345678::money; -ERROR: bigint out of range -SELECT 9223372036854775807::money; -ERROR: bigint out of range SELECT (-12345)::money; money ------------- @@ -220,10 +306,6 @@ SELECT (-12345678901234567)::money; -$12,345,678,901,234,567.00 (1 row) -SELECT (-123456789012345678)::money; -ERROR: bigint out of range -SELECT (-9223372036854775808)::money; -ERROR: bigint out of range SELECT 1234567890::int4::money; money ------------------- diff --git a/src/test/regress/sql/money.sql b/src/test/regress/sql/money.sql index 09b9476b70..561ccb527f 100644 --- a/src/test/regress/sql/money.sql +++ b/src/test/regress/sql/money.sql @@ -57,16 +57,38 @@ DELETE FROM money_data; INSERT INTO money_data VALUES ('$123.459'); SELECT * FROM money_data; +-- input checks +SELECT '1234567890'::money; +SELECT '12345678901234567'::money; +SELECT '123456789012345678'::money; +SELECT '9223372036854775807'::money; +SELECT '-12345'::money; +SELECT '-1234567890'::money; +SELECT '-12345678901234567'::money; +SELECT '-123456789012345678'::money; +SELECT '-9223372036854775808'::money; + +-- special characters +SELECT '(1)'::money; +SELECT '($123,456.78)'::money; + +-- documented minimums and maximums +SELECT '-92233720368547758.08'::money; +SELECT '92233720368547758.07'::money; + +SELECT '-92233720368547758.09'::money; +SELECT '92233720368547758.08'::money; + +-- rounding +SELECT '-92233720368547758.085'::money; +SELECT '92233720368547758.075'::money; + -- Cast int4/int8 to money SELECT 1234567890::money; SELECT 12345678901234567::money; -SELECT 123456789012345678::money; -SELECT 9223372036854775807::money; SELECT (-12345)::money; SELECT (-1234567890)::money; SELECT (-12345678901234567)::money; -SELECT (-123456789012345678)::money; -SELECT (-9223372036854775808)::money; SELECT 1234567890::int4::money; SELECT 12345678901234567::int8::money; SELECT (-1234567890)::int4::money; From c99dd5bfed23d9787dcf7d00197c1ed42bcfdb02 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 15 Sep 2016 11:51:43 +0300 Subject: [PATCH 173/871] Fix and clarify comments on replacement selection. These were modified by the patch to only use replacement selection for the first run in an external sort. --- src/backend/utils/sort/tuplesort.c | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index d600670d26..16ceb30b27 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -13,26 +13,26 @@ * See Knuth, volume 3, for more than you want to know about the external * sorting algorithm. Historically, we divided the input into sorted runs * using replacement selection, in the form of a priority tree implemented - * as a heap (essentially his Algorithm 5.2.3H -- although that strategy is - * often avoided altogether), but that can now only happen first the first - * run. We merge the runs using polyphase merge, Knuth's Algorithm + * as a heap (essentially his Algorithm 5.2.3H), but now we only do that + * for the first run, and only if the run would otherwise end up being very + * short. We merge the runs using polyphase merge, Knuth's Algorithm * 5.4.2D. The logical "tapes" used by Algorithm D are implemented by * logtape.c, which avoids space wastage by recycling disk space as soon * as each block is read from its "tape". * - * We never form the initial runs using Knuth's recommended replacement - * selection data structure (Algorithm 5.4.1R), because it uses a fixed - * number of records in memory at all times. Since we are dealing with - * tuples that may vary considerably in size, we want to be able to vary - * the number of records kept in memory to ensure full utilization of the - * allowed sort memory space. So, we keep the tuples in a variable-size - * heap, with the next record to go out at the top of the heap. Like - * Algorithm 5.4.1R, each record is stored with the run number that it - * must go into, and we use (run number, key) as the ordering key for the - * heap. When the run number at the top of the heap changes, we know that - * no more records of the prior run are left in the heap. Note that there - * are in practice only ever two distinct run numbers, due to the greatly - * reduced use of replacement selection in PostgreSQL 9.6. + * We do not use Knuth's recommended data structure (Algorithm 5.4.1R) for + * the replacement selection, because it uses a fixed number of records + * in memory at all times. Since we are dealing with tuples that may vary + * considerably in size, we want to be able to vary the number of records + * kept in memory to ensure full utilization of the allowed sort memory + * space. So, we keep the tuples in a variable-size heap, with the next + * record to go out at the top of the heap. Like Algorithm 5.4.1R, each + * record is stored with the run number that it must go into, and we use + * (run number, key) as the ordering key for the heap. When the run number + * at the top of the heap changes, we know that no more records of the prior + * run are left in the heap. Note that there are in practice only ever two + * distinct run numbers, because since PostgreSQL 9.6, we only use + * replacement selection to form the first run. * * In PostgreSQL 9.6, a heap (based on Knuth's Algorithm H, with some small * customizations) is only used with the aim of producing just one run, From 593d4e47db7af1a3a5dd6b6b1971f181b5566dbd Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 15 Sep 2016 12:36:21 +0300 Subject: [PATCH 174/871] Support OpenSSL 1.1.0. Changes needed to build at all: - Check for SSL_new in configure, now that SSL_library_init is a macro. - Do not access struct members directly. This includes some new code in pgcrypto, to use the resource owner mechanism to ensure that we don't leak OpenSSL handles, now that we can't embed them in other structs anymore. - RAND_SSLeay() -> RAND_OpenSSL() Changes that were needed to silence deprecation warnings, but were not strictly necessary: - RAND_pseudo_bytes() -> RAND_bytes(). - SSL_library_init() and OpenSSL_config() -> OPENSSL_init_ssl() - ASN1_STRING_data() -> ASN1_STRING_get0_data() - DH_generate_parameters() -> DH_generate_parameters() - Locking callbacks are not needed with OpenSSL 1.1.0 anymore. (Good riddance!) Also change references to SSLEAY_VERSION_NUMBER with OPENSSL_VERSION_NUMBER, for the sake of consistency. OPENSSL_VERSION_NUMBER has existed since time immemorial. Fix SSL test suite to work with OpenSSL 1.1.0. CA certificates must have the "CA:true" basic constraint extension now, or OpenSSL will refuse them. Regenerate the test certificates with that. The "openssl" binary, used to generate the certificates, is also now more picky, and throws an error if an X509 extension is specified in "req_extensions", but that section is empty. Backpatch to all supported branches, per popular demand. In back-branches, we still support OpenSSL 0.9.7 and above. OpenSSL 0.9.6 should still work too, but I didn't test it. In master, we only support 0.9.8 and above. Patch by Andreas Karlsson, with additional changes by me. Discussion: <20160627151604.GD1051@msg.df7cb.de> --- configure | 44 +++--- configure.in | 4 +- contrib/pgcrypto/internal.c | 9 -- contrib/pgcrypto/openssl.c | 130 ++++++++++++++---- contrib/pgcrypto/pgcrypto.c | 2 +- contrib/pgcrypto/pgp-s2k.c | 6 +- contrib/pgcrypto/px-crypt.c | 2 +- contrib/pgcrypto/px.h | 1 - contrib/sslinfo/sslinfo.c | 14 +- src/backend/libpq/be-secure-openssl.c | 93 +++++++++++-- src/interfaces/libpq/fe-secure-openssl.c | 94 ++++++++++--- src/test/ssl/Makefile | 5 +- src/test/ssl/cas.config | 7 +- src/test/ssl/root_ca.config | 4 + src/test/ssl/server-cn-only.config | 1 - src/test/ssl/server-no-names.config | 1 - src/test/ssl/server-revoked.config | 1 - src/test/ssl/ssl/both-cas-1.crt | 67 ++++----- src/test/ssl/ssl/both-cas-2.crt | 67 ++++----- src/test/ssl/ssl/client-revoked.crt | 16 +-- src/test/ssl/ssl/client-revoked.key | 26 ++-- src/test/ssl/ssl/client.crl | 12 +- src/test/ssl/ssl/client.crt | 16 +-- src/test/ssl/ssl/client.key | 26 ++-- src/test/ssl/ssl/client_ca.crt | 22 +-- src/test/ssl/ssl/client_ca.key | 26 ++-- src/test/ssl/ssl/root+client.crl | 22 +-- src/test/ssl/ssl/root+client_ca.crt | 45 +++--- src/test/ssl/ssl/root+server.crl | 22 +-- src/test/ssl/ssl/root+server_ca.crt | 45 +++--- src/test/ssl/ssl/root.crl | 10 +- src/test/ssl/ssl/root_ca.crt | 23 ++-- src/test/ssl/ssl/root_ca.key | 26 ++-- src/test/ssl/ssl/server-cn-and-alt-names.crt | 18 +-- src/test/ssl/ssl/server-cn-and-alt-names.key | 26 ++-- src/test/ssl/ssl/server-cn-only.crt | 16 +-- src/test/ssl/ssl/server-cn-only.key | 26 ++-- .../ssl/ssl/server-multiple-alt-names.crt | 16 +-- .../ssl/ssl/server-multiple-alt-names.key | 26 ++-- src/test/ssl/ssl/server-no-names.crt | 14 +- src/test/ssl/ssl/server-no-names.key | 26 ++-- src/test/ssl/ssl/server-revoked.crt | 16 +-- src/test/ssl/ssl/server-revoked.key | 26 ++-- src/test/ssl/ssl/server-single-alt-name.crt | 14 +- src/test/ssl/ssl/server-single-alt-name.key | 26 ++-- src/test/ssl/ssl/server-ss.crt | 16 +-- src/test/ssl/ssl/server-ss.key | 26 ++-- src/test/ssl/ssl/server.crl | 12 +- src/test/ssl/ssl/server_ca.crt | 22 +-- src/test/ssl/ssl/server_ca.key | 26 ++-- 50 files changed, 707 insertions(+), 534 deletions(-) diff --git a/configure b/configure index 45c8eefad7..caf6f260ee 100755 --- a/configure +++ b/configure @@ -9538,9 +9538,9 @@ else as_fn_error $? "library 'crypto' is required for OpenSSL" "$LINENO" 5 fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_library_init in -lssl" >&5 -$as_echo_n "checking for SSL_library_init in -lssl... " >&6; } -if ${ac_cv_lib_ssl_SSL_library_init+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_new in -lssl" >&5 +$as_echo_n "checking for SSL_new in -lssl... " >&6; } +if ${ac_cv_lib_ssl_SSL_new+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -9554,27 +9554,27 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char SSL_library_init (); +char SSL_new (); int main () { -return SSL_library_init (); +return SSL_new (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_ssl_SSL_library_init=yes + ac_cv_lib_ssl_SSL_new=yes else - ac_cv_lib_ssl_SSL_library_init=no + ac_cv_lib_ssl_SSL_new=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_library_init" >&5 -$as_echo "$ac_cv_lib_ssl_SSL_library_init" >&6; } -if test "x$ac_cv_lib_ssl_SSL_library_init" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_new" >&5 +$as_echo "$ac_cv_lib_ssl_SSL_new" >&6; } +if test "x$ac_cv_lib_ssl_SSL_new" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_LIBSSL 1 _ACEOF @@ -9644,9 +9644,9 @@ else as_fn_error $? "library 'eay32' or 'crypto' is required for OpenSSL" "$LINENO" 5 fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing SSL_library_init" >&5 -$as_echo_n "checking for library containing SSL_library_init... " >&6; } -if ${ac_cv_search_SSL_library_init+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing SSL_new" >&5 +$as_echo_n "checking for library containing SSL_new... " >&6; } +if ${ac_cv_search_SSL_new+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS @@ -9659,11 +9659,11 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char SSL_library_init (); +char SSL_new (); int main () { -return SSL_library_init (); +return SSL_new (); ; return 0; } @@ -9676,25 +9676,25 @@ for ac_lib in '' ssleay32 ssl; do LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_SSL_library_init=$ac_res + ac_cv_search_SSL_new=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext - if ${ac_cv_search_SSL_library_init+:} false; then : + if ${ac_cv_search_SSL_new+:} false; then : break fi done -if ${ac_cv_search_SSL_library_init+:} false; then : +if ${ac_cv_search_SSL_new+:} false; then : else - ac_cv_search_SSL_library_init=no + ac_cv_search_SSL_new=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_SSL_library_init" >&5 -$as_echo "$ac_cv_search_SSL_library_init" >&6; } -ac_res=$ac_cv_search_SSL_library_init +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_SSL_new" >&5 +$as_echo "$ac_cv_search_SSL_new" >&6; } +ac_res=$ac_cv_search_SSL_new if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" diff --git a/configure.in b/configure.in index c878b4e377..c42680607a 100644 --- a/configure.in +++ b/configure.in @@ -1112,10 +1112,10 @@ if test "$with_openssl" = yes ; then dnl Order matters! if test "$PORTNAME" != "win32"; then AC_CHECK_LIB(crypto, CRYPTO_new_ex_data, [], [AC_MSG_ERROR([library 'crypto' is required for OpenSSL])]) - AC_CHECK_LIB(ssl, SSL_library_init, [], [AC_MSG_ERROR([library 'ssl' is required for OpenSSL])]) + AC_CHECK_LIB(ssl, SSL_new, [], [AC_MSG_ERROR([library 'ssl' is required for OpenSSL])]) else AC_SEARCH_LIBS(CRYPTO_new_ex_data, eay32 crypto, [], [AC_MSG_ERROR([library 'eay32' or 'crypto' is required for OpenSSL])]) - AC_SEARCH_LIBS(SSL_library_init, ssleay32 ssl, [], [AC_MSG_ERROR([library 'ssleay32' or 'ssl' is required for OpenSSL])]) + AC_SEARCH_LIBS(SSL_new, ssleay32 ssl, [], [AC_MSG_ERROR([library 'ssleay32' or 'ssl' is required for OpenSSL])]) fi AC_CHECK_FUNCS([SSL_get_current_compression]) fi diff --git a/contrib/pgcrypto/internal.c b/contrib/pgcrypto/internal.c index cb8ba2633d..02ff976c25 100644 --- a/contrib/pgcrypto/internal.c +++ b/contrib/pgcrypto/internal.c @@ -620,15 +620,6 @@ px_find_cipher(const char *name, PX_Cipher **res) * Randomness provider */ -/* - * Use always strong randomness. - */ -int -px_get_pseudo_random_bytes(uint8 *dst, unsigned count) -{ - return px_get_random_bytes(dst, count); -} - static time_t seed_time = 0; static time_t check_time = 0; diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index ffab5d2bb0..e264aa49df 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -41,6 +41,9 @@ #include #include +#include "utils/memutils.h" +#include "utils/resowner.h" + /* * Max lengths we might want to handle. */ @@ -51,18 +54,73 @@ * Hashes */ +/* + * To make sure we don't leak OpenSSL handles on abort, we keep OSSLDigest + * objects in a linked list, allocated in TopMemoryContext. We use the + * ResourceOwner mechanism to free them on abort. + */ typedef struct OSSLDigest { const EVP_MD *algo; - EVP_MD_CTX ctx; + EVP_MD_CTX *ctx; + + ResourceOwner owner; + struct OSSLDigest *next; + struct OSSLDigest *prev; } OSSLDigest; +static OSSLDigest *open_digests = NULL; +static bool resowner_callback_registered = false; + +static void +free_openssldigest(OSSLDigest *digest) +{ + EVP_MD_CTX_destroy(digest->ctx); + if (digest->prev) + digest->prev->next = digest->next; + else + open_digests = digest->next; + if (digest->next) + digest->next->prev = digest->prev; + pfree(digest); +} + +/* + * Close any open OpenSSL handles on abort. + */ +static void +digest_free_callback(ResourceReleasePhase phase, + bool isCommit, + bool isTopLevel, + void *arg) +{ + OSSLDigest *curr; + OSSLDigest *next; + + if (phase != RESOURCE_RELEASE_AFTER_LOCKS) + return; + + next = open_digests; + while (next) + { + curr = next; + next = curr->next; + + if (curr->owner == CurrentResourceOwner) + { + if (isCommit) + elog(WARNING, "pgcrypto digest reference leak: digest %p still referenced", curr); + free_openssldigest(curr); + } + } +} + static unsigned digest_result_size(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - return EVP_MD_CTX_size(&digest->ctx); + return EVP_MD_CTX_size(digest->ctx); } static unsigned @@ -70,7 +128,7 @@ digest_block_size(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - return EVP_MD_CTX_block_size(&digest->ctx); + return EVP_MD_CTX_block_size(digest->ctx); } static void @@ -78,7 +136,7 @@ digest_reset(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_DigestInit_ex(&digest->ctx, digest->algo, NULL); + EVP_DigestInit_ex(digest->ctx, digest->algo, NULL); } static void @@ -86,7 +144,7 @@ digest_update(PX_MD *h, const uint8 *data, unsigned dlen) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_DigestUpdate(&digest->ctx, data, dlen); + EVP_DigestUpdate(digest->ctx, data, dlen); } static void @@ -94,7 +152,7 @@ digest_finish(PX_MD *h, uint8 *dst) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_DigestFinal_ex(&digest->ctx, dst, NULL); + EVP_DigestFinal_ex(digest->ctx, dst, NULL); } static void @@ -102,9 +160,7 @@ digest_free(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_MD_CTX_cleanup(&digest->ctx); - - px_free(digest); + free_openssldigest(digest); px_free(h); } @@ -116,6 +172,7 @@ int px_find_digest(const char *name, PX_MD **res) { const EVP_MD *md; + EVP_MD_CTX *ctx; PX_MD *h; OSSLDigest *digest; @@ -125,17 +182,43 @@ px_find_digest(const char *name, PX_MD **res) OpenSSL_add_all_algorithms(); } + if (!resowner_callback_registered) + { + RegisterResourceReleaseCallback(digest_free_callback, NULL); + resowner_callback_registered = true; + } + md = EVP_get_digestbyname(name); if (md == NULL) return PXE_NO_HASH; - digest = px_alloc(sizeof(*digest)); - digest->algo = md; + /* + * Create an OSSLDigest object, an OpenSSL MD object, and a PX_MD object. + * The order is crucial, to make sure we don't leak anything on + * out-of-memory or other error. + */ + digest = MemoryContextAlloc(TopMemoryContext, sizeof(*digest)); - EVP_MD_CTX_init(&digest->ctx); - if (EVP_DigestInit_ex(&digest->ctx, digest->algo, NULL) == 0) + ctx = EVP_MD_CTX_create(); + if (!ctx) + { + pfree(digest); + return -1; + } + if (EVP_DigestInit_ex(ctx, md, NULL) == 0) + { + pfree(digest); return -1; + } + digest->algo = md; + digest->ctx = ctx; + digest->owner = CurrentResourceOwner; + digest->next = open_digests; + digest->prev = NULL; + open_digests = digest; + + /* The PX_MD object is allocated in the current memory context. */ h = px_alloc(sizeof(*h)); h->result_size = digest_result_size; h->block_size = digest_block_size; @@ -831,6 +914,10 @@ px_find_cipher(const char *name, PX_Cipher **res) static int openssl_random_init = 0; +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define RAND_OpenSSL RAND_SSLeay +#endif + /* * OpenSSL random should re-feeded occasionally. From /dev/urandom * preferably. @@ -839,7 +926,7 @@ static void init_openssl_rand(void) { if (RAND_get_rand_method() == NULL) - RAND_set_rand_method(RAND_SSLeay()); + RAND_set_rand_method(RAND_OpenSSL()); openssl_random_init = 1; } @@ -858,21 +945,6 @@ px_get_random_bytes(uint8 *dst, unsigned count) return PXE_OSSL_RAND_ERROR; } -int -px_get_pseudo_random_bytes(uint8 *dst, unsigned count) -{ - int res; - - if (!openssl_random_init) - init_openssl_rand(); - - res = RAND_pseudo_bytes(dst, count); - if (res == 0 || res == 1) - return count; - - return PXE_OSSL_RAND_ERROR; -} - int px_add_entropy(const uint8 *data, unsigned count) { diff --git a/contrib/pgcrypto/pgcrypto.c b/contrib/pgcrypto/pgcrypto.c index 2d446d8cc9..27b96c7cc4 100644 --- a/contrib/pgcrypto/pgcrypto.c +++ b/contrib/pgcrypto/pgcrypto.c @@ -454,7 +454,7 @@ pg_random_uuid(PG_FUNCTION_ARGS) int err; /* generate random bits */ - err = px_get_pseudo_random_bytes(buf, UUID_LEN); + err = px_get_random_bytes(buf, UUID_LEN); if (err < 0) ereport(ERROR, (errcode(ERRCODE_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION), diff --git a/contrib/pgcrypto/pgp-s2k.c b/contrib/pgcrypto/pgp-s2k.c index 9937d154f2..3551d44d62 100644 --- a/contrib/pgcrypto/pgp-s2k.c +++ b/contrib/pgcrypto/pgp-s2k.c @@ -233,13 +233,13 @@ pgp_s2k_fill(PGP_S2K *s2k, int mode, int digest_algo, int count) case PGP_S2K_SIMPLE: break; case PGP_S2K_SALTED: - res = px_get_pseudo_random_bytes(s2k->salt, PGP_S2K_SALT); + res = px_get_random_bytes(s2k->salt, PGP_S2K_SALT); break; case PGP_S2K_ISALTED: - res = px_get_pseudo_random_bytes(s2k->salt, PGP_S2K_SALT); + res = px_get_random_bytes(s2k->salt, PGP_S2K_SALT); if (res < 0) break; - res = px_get_pseudo_random_bytes(&tmp, 1); + res = px_get_random_bytes(&tmp, 1); if (res < 0) break; s2k->iter = decide_s2k_iter(tmp, count); diff --git a/contrib/pgcrypto/px-crypt.c b/contrib/pgcrypto/px-crypt.c index e3246fc5b9..3d42393850 100644 --- a/contrib/pgcrypto/px-crypt.c +++ b/contrib/pgcrypto/px-crypt.c @@ -153,7 +153,7 @@ px_gen_salt(const char *salt_type, char *buf, int rounds) return PXE_BAD_SALT_ROUNDS; } - res = px_get_pseudo_random_bytes((uint8 *) rbuf, g->input_len); + res = px_get_random_bytes((uint8 *) rbuf, g->input_len); if (res < 0) return res; diff --git a/contrib/pgcrypto/px.h b/contrib/pgcrypto/px.h index 0f6bbd7a8d..9174e137db 100644 --- a/contrib/pgcrypto/px.h +++ b/contrib/pgcrypto/px.h @@ -190,7 +190,6 @@ int px_find_cipher(const char *name, PX_Cipher **res); int px_find_combo(const char *name, PX_Combo **res); int px_get_random_bytes(uint8 *dst, unsigned count); -int px_get_pseudo_random_bytes(uint8 *dst, unsigned count); int px_add_entropy(const uint8 *data, unsigned count); unsigned px_acquire_system_randomness(uint8 *dst); diff --git a/contrib/sslinfo/sslinfo.c b/contrib/sslinfo/sslinfo.c index 82a4c1bd70..a4b0f9b6a1 100644 --- a/contrib/sslinfo/sslinfo.c +++ b/contrib/sslinfo/sslinfo.c @@ -402,8 +402,6 @@ ssl_extension_info(PG_FUNCTION_ARGS) MemoryContext oldcontext; SSLExtensionInfoContext *fctx; - STACK_OF(X509_EXTENSION) *ext_stack = NULL; - if (SRF_IS_FIRSTCALL()) { @@ -427,16 +425,10 @@ ssl_extension_info(PG_FUNCTION_ARGS) errmsg("function returning record called in context that cannot accept type record"))); fctx->tupdesc = BlessTupleDesc(tupdesc); - /* Get all extensions of certificate */ - if (cert && cert->cert_info) - ext_stack = cert->cert_info->extensions; - /* Set max_calls as a count of extensions in certificate */ max_calls = cert != NULL ? X509_get_ext_count(cert) : 0; - if (cert != NULL && - ext_stack != NULL && - max_calls > 0) + if (max_calls > 0) { /* got results, keep track of them */ funcctx->max_calls = max_calls; @@ -462,8 +454,6 @@ ssl_extension_info(PG_FUNCTION_ARGS) max_calls = funcctx->max_calls; fctx = funcctx->user_fctx; - ext_stack = cert->cert_info->extensions; - /* do while there are more left to send */ if (call_cntr < max_calls) { @@ -486,7 +476,7 @@ ssl_extension_info(PG_FUNCTION_ARGS) errmsg("could not create OpenSSL BIO structure"))); /* Get the extension from the certificate */ - ext = sk_X509_EXTENSION_value(ext_stack, call_cntr); + ext = X509_get_ext(cert, call_cntr); obj = X509_EXTENSION_get_object(ext); /* Get the extension name */ diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index bb0d2d977f..2afd738945 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -72,6 +72,7 @@ static int my_SSL_set_fd(Port *port, int fd); static DH *load_dh_file(int keylength); static DH *load_dh_buffer(const char *, size_t); +static DH *generate_dh_parameters(int prime_len, int generator); static DH *tmp_dh_cb(SSL *s, int is_export, int keylength); static int verify_cb(int, X509_STORE_CTX *); static void info_cb(const SSL *ssl, int type, int args); @@ -164,9 +165,13 @@ be_tls_init(void) if (!SSL_context) { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL); +#else OPENSSL_config(NULL); SSL_library_init(); SSL_load_error_strings(); +#endif /* * We use SSLv23_method() because it can negotiate use of the highest @@ -667,8 +672,12 @@ be_tls_write(Port *port, void *ptr, size_t len, int *waitfor) * to retry; do we need to adopt their logic for that? */ -static bool my_bio_initialized = false; -static BIO_METHOD my_bio_methods; +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define BIO_get_data(bio) (bio->ptr) +#define BIO_set_data(bio, data) (bio->ptr = data) +#endif + +static BIO_METHOD *my_bio_methods = NULL; static int my_sock_read(BIO *h, char *buf, int size) @@ -677,7 +686,7 @@ my_sock_read(BIO *h, char *buf, int size) if (buf != NULL) { - res = secure_raw_read(((Port *) h->ptr), buf, size); + res = secure_raw_read(((Port *) BIO_get_data(h)), buf, size); BIO_clear_retry_flags(h); if (res <= 0) { @@ -697,7 +706,7 @@ my_sock_write(BIO *h, const char *buf, int size) { int res = 0; - res = secure_raw_write(((Port *) h->ptr), buf, size); + res = secure_raw_write(((Port *) BIO_get_data(h)), buf, size); BIO_clear_retry_flags(h); if (res <= 0) { @@ -714,14 +723,41 @@ my_sock_write(BIO *h, const char *buf, int size) static BIO_METHOD * my_BIO_s_socket(void) { - if (!my_bio_initialized) + if (!my_bio_methods) { - memcpy(&my_bio_methods, BIO_s_socket(), sizeof(BIO_METHOD)); - my_bio_methods.bread = my_sock_read; - my_bio_methods.bwrite = my_sock_write; - my_bio_initialized = true; + BIO_METHOD *biom = (BIO_METHOD *) BIO_s_socket(); +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + int my_bio_index; + + my_bio_index = BIO_get_new_index(); + if (my_bio_index == -1) + return NULL; + my_bio_methods = BIO_meth_new(my_bio_index, "PostgreSQL backend socket"); + if (!my_bio_methods) + return NULL; + if (!BIO_meth_set_write(my_bio_methods, my_sock_write) || + !BIO_meth_set_read(my_bio_methods, my_sock_read) || + !BIO_meth_set_gets(my_bio_methods, BIO_meth_get_gets(biom)) || + !BIO_meth_set_puts(my_bio_methods, BIO_meth_get_puts(biom)) || + !BIO_meth_set_ctrl(my_bio_methods, BIO_meth_get_ctrl(biom)) || + !BIO_meth_set_create(my_bio_methods, BIO_meth_get_create(biom)) || + !BIO_meth_set_destroy(my_bio_methods, BIO_meth_get_destroy(biom)) || + !BIO_meth_set_callback_ctrl(my_bio_methods, BIO_meth_get_callback_ctrl(biom))) + { + BIO_meth_free(my_bio_methods); + my_bio_methods = NULL; + return NULL; + } +#else + my_bio_methods = malloc(sizeof(BIO_METHOD)); + if (!my_bio_methods) + return NULL; + memcpy(my_bio_methods, biom, sizeof(BIO_METHOD)); + my_bio_methods->bread = my_sock_read; + my_bio_methods->bwrite = my_sock_write; +#endif } - return &my_bio_methods; + return my_bio_methods; } /* This should exactly match openssl's SSL_set_fd except for using my BIO */ @@ -729,17 +765,23 @@ static int my_SSL_set_fd(Port *port, int fd) { int ret = 0; - BIO *bio = NULL; + BIO *bio; + BIO_METHOD *bio_method; - bio = BIO_new(my_BIO_s_socket()); + bio_method = my_BIO_s_socket(); + if (bio_method == NULL) + { + SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB); + goto err; + } + bio = BIO_new(bio_method); if (bio == NULL) { SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB); goto err; } - /* Use 'ptr' to store pointer to PGconn */ - bio->ptr = port; + BIO_set_data(bio, port); BIO_set_fd(bio, fd, BIO_NOCLOSE); SSL_set_bio(port->ssl, bio, bio); @@ -833,6 +875,27 @@ load_dh_buffer(const char *buffer, size_t len) return dh; } +/* + * Generate DH parameters. + * + * Last resort if we can't load precomputed nor hardcoded + * parameters. + */ +static DH * +generate_dh_parameters(int prime_len, int generator) +{ + DH *dh; + + if ((dh = DH_new()) == NULL) + return NULL; + + if (DH_generate_parameters_ex(dh, prime_len, generator, NULL)) + return dh; + + DH_free(dh); + return NULL; +} + /* * Generate an ephemeral DH key. Because this can take a long * time to compute, we can use precomputed parameters of the @@ -902,7 +965,7 @@ tmp_dh_cb(SSL *s, int is_export, int keylength) ereport(DEBUG2, (errmsg_internal("DH: generating parameters (%d bits)", keylength))); - r = DH_generate_parameters(keylength, DH_GENERATOR_2, NULL, NULL); + r = generate_dh_parameters(keylength, DH_GENERATOR_2); } return r; diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index d8716128ec..fe81825144 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -506,6 +506,9 @@ wildcard_certificate_match(const char *pattern, const char *string) return 1; } +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define ASN1_STRING_get0_data ASN1_STRING_data +#endif /* * Check if a name from a server's certificate matches the peer's hostname. @@ -522,7 +525,7 @@ verify_peer_name_matches_certificate_name(PGconn *conn, ASN1_STRING *name_entry, { int len; char *name; - unsigned char *namedata; + const unsigned char *namedata; int result; *store_name = NULL; @@ -541,7 +544,7 @@ verify_peer_name_matches_certificate_name(PGconn *conn, ASN1_STRING *name_entry, * There is no guarantee the string returned from the certificate is * NULL-terminated, so make a copy that is. */ - namedata = ASN1_STRING_data(name_entry); + namedata = ASN1_STRING_get0_data(name_entry); len = ASN1_STRING_length(name_entry); name = malloc(len + 1); if (name == NULL) @@ -729,9 +732,10 @@ verify_peer_name_matches_certificate(PGconn *conn) return found_match && !got_error; } -#ifdef ENABLE_THREAD_SAFETY +#if defined(ENABLE_THREAD_SAFETY) && OPENSSL_VERSION_NUMBER < 0x10100000L /* - * Callback functions for OpenSSL internal locking + * Callback functions for OpenSSL internal locking. (OpenSSL 1.1.0 + * does its own locking, and doesn't need these anymore.) */ static unsigned long @@ -761,7 +765,7 @@ pq_lockingcallback(int mode, int n, const char *file, int line) PGTHREAD_ERROR("failed to unlock mutex"); } } -#endif /* ENABLE_THREAD_SAFETY */ +#endif /* ENABLE_THREAD_SAFETY && OPENSSL_VERSION_NUMBER < 0x10100000L */ /* * Initialize SSL system, in particular creating the SSL_context object @@ -800,6 +804,7 @@ pgtls_init(PGconn *conn) if (pthread_mutex_lock(&ssl_config_mutex)) return -1; +#if OPENSSL_VERSION_NUMBER < 0x10100000L if (pq_init_crypto_lib) { /* @@ -840,15 +845,20 @@ pgtls_init(PGconn *conn) CRYPTO_set_locking_callback(pq_lockingcallback); } } +#endif /* OPENSSL_VERSION_NUMBER < 0x10100000L */ #endif /* ENABLE_THREAD_SAFETY */ if (!SSL_context) { if (pq_init_ssl_lib) { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL); +#else OPENSSL_config(NULL); SSL_library_init(); SSL_load_error_strings(); +#endif } /* @@ -897,12 +907,13 @@ pgtls_init(PGconn *conn) * if we had any.) * * Callbacks are only set when we're compiled in threadsafe mode, so - * we only need to remove them in this case. + * we only need to remove them in this case. They are also not needed + * with OpenSSL 1.1.0 anymore. */ static void destroy_ssl_system(void) { -#ifdef ENABLE_THREAD_SAFETY +#if defined(ENABLE_THREAD_SAFETY) && OPENSSL_VERSION_NUMBER < 0x10100000L /* Mutex is created in initialize_ssl_system() */ if (pthread_mutex_lock(&ssl_config_mutex)) return; @@ -1617,15 +1628,19 @@ PQsslAttribute(PGconn *conn, const char *attribute_name) * to retry; do we need to adopt their logic for that? */ -static bool my_bio_initialized = false; -static BIO_METHOD my_bio_methods; +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define BIO_get_data(bio) (bio->ptr) +#define BIO_set_data(bio, data) (bio->ptr = data) +#endif + +static BIO_METHOD *my_bio_methods; static int my_sock_read(BIO *h, char *buf, int size) { int res; - res = pqsecure_raw_read((PGconn *) h->ptr, buf, size); + res = pqsecure_raw_read((PGconn *) BIO_get_data(h), buf, size); BIO_clear_retry_flags(h); if (res < 0) { @@ -1655,7 +1670,7 @@ my_sock_write(BIO *h, const char *buf, int size) { int res; - res = pqsecure_raw_write((PGconn *) h->ptr, buf, size); + res = pqsecure_raw_write((PGconn *) BIO_get_data(h), buf, size); BIO_clear_retry_flags(h); if (res <= 0) { @@ -1683,14 +1698,45 @@ my_sock_write(BIO *h, const char *buf, int size) static BIO_METHOD * my_BIO_s_socket(void) { - if (!my_bio_initialized) + if (!my_bio_methods) { - memcpy(&my_bio_methods, BIO_s_socket(), sizeof(BIO_METHOD)); - my_bio_methods.bread = my_sock_read; - my_bio_methods.bwrite = my_sock_write; - my_bio_initialized = true; + BIO_METHOD *biom = (BIO_METHOD *) BIO_s_socket(); +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + int my_bio_index; + + my_bio_index = BIO_get_new_index(); + if (my_bio_index == -1) + return NULL; + my_bio_methods = BIO_meth_new(my_bio_index, "libpq socket"); + if (!my_bio_methods) + return NULL; + /* + * As of this writing, these functions never fail. But check anyway, like + * OpenSSL's own examples do. + */ + if (!BIO_meth_set_write(my_bio_methods, my_sock_write) || + !BIO_meth_set_read(my_bio_methods, my_sock_read) || + !BIO_meth_set_gets(my_bio_methods, BIO_meth_get_gets(biom)) || + !BIO_meth_set_puts(my_bio_methods, BIO_meth_get_puts(biom)) || + !BIO_meth_set_ctrl(my_bio_methods, BIO_meth_get_ctrl(biom)) || + !BIO_meth_set_create(my_bio_methods, BIO_meth_get_create(biom)) || + !BIO_meth_set_destroy(my_bio_methods, BIO_meth_get_destroy(biom)) || + !BIO_meth_set_callback_ctrl(my_bio_methods, BIO_meth_get_callback_ctrl(biom))) + { + BIO_meth_free(my_bio_methods); + my_bio_methods = NULL; + return NULL; + } +#else + my_bio_methods = malloc(sizeof(BIO_METHOD)); + if (!my_bio_methods) + return NULL; + memcpy(my_bio_methods, biom, sizeof(BIO_METHOD)); + my_bio_methods->bread = my_sock_read; + my_bio_methods->bwrite = my_sock_write; +#endif } - return &my_bio_methods; + return my_bio_methods; } /* This should exactly match openssl's SSL_set_fd except for using my BIO */ @@ -1698,16 +1744,22 @@ static int my_SSL_set_fd(PGconn *conn, int fd) { int ret = 0; - BIO *bio = NULL; + BIO *bio; + BIO_METHOD *bio_method; - bio = BIO_new(my_BIO_s_socket()); + bio_method = my_BIO_s_socket(); + if (bio_method == NULL) + { + SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB); + goto err; + } + bio = BIO_new(bio_method); if (bio == NULL) { SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB); goto err; } - /* Use 'ptr' to store pointer to PGconn */ - bio->ptr = conn; + BIO_set_data(bio, conn); SSL_set_bio(conn->ssl, bio, bio); BIO_set_fd(bio, fd, BIO_NOCLOSE); diff --git a/src/test/ssl/Makefile b/src/test/ssl/Makefile index d8c4741963..3d992babff 100644 --- a/src/test/ssl/Makefile +++ b/src/test/ssl/Makefile @@ -41,15 +41,16 @@ ssl/%.key: # Root CA certificate ssl/root_ca.crt: ssl/root_ca.key cas.config touch ssl/root_ca-certindex - openssl req -new -out ssl/root_ca.crt -x509 -config cas.config -config root_ca.config -key ssl/root_ca.key -days 10000 + openssl req -new -out ssl/root_ca.crt -x509 -config cas.config -config root_ca.config -key ssl/root_ca.key -days 10000 -extensions v3_ca echo "01" > ssl/root_ca.srl # Client and server CAs ssl/%_ca.crt: ssl/%_ca.key %_ca.config ssl/root_ca.crt ssl/new_certs_dir touch ssl/$*_ca-certindex + echo "unique_subject=no" > ssl/$*_ca-certindex.attr openssl req -new -out ssl/temp_ca.crt -config cas.config -config $*_ca.config -key ssl/$*_ca.key # Sign the certificate with the root CA - openssl ca -name root_ca -batch -config cas.config -in ssl/temp_ca.crt -out ssl/temp_ca_signed.crt + openssl ca -name root_ca -batch -config cas.config -in ssl/temp_ca.crt -out ssl/temp_ca_signed.crt -extensions v3_ca openssl x509 -in ssl/temp_ca_signed.crt -out ssl/$*_ca.crt # to keep just the PEM cert rm ssl/temp_ca.crt ssl/temp_ca_signed.crt echo "01" > ssl/$*_ca.srl diff --git a/src/test/ssl/cas.config b/src/test/ssl/cas.config index 9c6cbb93f3..013cebae16 100644 --- a/src/test/ssl/cas.config +++ b/src/test/ssl/cas.config @@ -2,11 +2,10 @@ [ req ] prompt = no -req_extensions = v3_req -# For Subject Alternative Names -[ v3_req ] -subjectAltName = @alt_names +# Extensions for CA certs +[ v3_ca ] +basicConstraints = CA:true # Root CA, used to sign the certificates of the intermediary server and # client CAs. diff --git a/src/test/ssl/root_ca.config b/src/test/ssl/root_ca.config index 72484d040f..187a9b8a46 100644 --- a/src/test/ssl/root_ca.config +++ b/src/test/ssl/root_ca.config @@ -7,3 +7,7 @@ prompt = no [ req_distinguished_name ] CN = Test root CA for PostgreSQL SSL regression test suite + +# Extensions for CA certs +[ v3_ca ] +basicConstraints = CA:true diff --git a/src/test/ssl/server-cn-only.config b/src/test/ssl/server-cn-only.config index 7849d090eb..1e5d582213 100644 --- a/src/test/ssl/server-cn-only.config +++ b/src/test/ssl/server-cn-only.config @@ -3,7 +3,6 @@ [ req ] distinguished_name = req_distinguished_name -req_extensions = v3_req prompt = no [ req_distinguished_name ] diff --git a/src/test/ssl/server-no-names.config b/src/test/ssl/server-no-names.config index 92cf779086..2a71125a42 100644 --- a/src/test/ssl/server-no-names.config +++ b/src/test/ssl/server-no-names.config @@ -5,7 +5,6 @@ [ req ] distinguished_name = req_distinguished_name -req_extensions = v3_req prompt = no [ req_distinguished_name ] diff --git a/src/test/ssl/server-revoked.config b/src/test/ssl/server-revoked.config index 9924af8ec1..47ef6462dc 100644 --- a/src/test/ssl/server-revoked.config +++ b/src/test/ssl/server-revoked.config @@ -5,7 +5,6 @@ [ req ] distinguished_name = req_distinguished_name -req_extensions = v3_req prompt = no [ req_distinguished_name ] diff --git a/src/test/ssl/ssl/both-cas-1.crt b/src/test/ssl/ssl/both-cas-1.crt index abf4612f9f..0e2a10a180 100644 --- a/src/test/ssl/ssl/both-cas-1.crt +++ b/src/test/ssl/ssl/both-cas-1.crt @@ -1,39 +1,40 @@ -----BEGIN CERTIFICATE----- -MIIB9zCCAWACCQDrgvp38CAy8DANBgkqhkiG9w0BAQsFADBAMT4wPAYDVQQDDDVU -ZXN0IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBz -dWl0ZTAeFw0xNTAyMTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMEAxPjA8BgNVBAMM -NVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0 -IHN1aXRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCyTfGMPAjAylLr3G7c -/QToCA3da5YZzdhd3TiQGugrJjWI4TzVB7pQ8IwDYk/jZf5TzVdEtz0B4TeIeUZl -FLW9dMpa/8SY2TETvMTuXR5MOxyw6FMEKb3buolsIksCCQ1btEIrDZ+gv9SJXcdL -ylU+VI1lKmn2fLNWWATzWrIUawIDAQABMA0GCSqGSIb3DQEBCwUAA4GBAF2T84iG -zWKXu+3PysuPOn7RuRpMgYQKouQktErNJ8hM7Yqj3vu879zUkX1rP0HGnx7xQC3d -nBkoJ7yNDR0MwQpWo1Dj1HLKNEY6ojKJgPd0+m8nG+02yUmmOjo0oMYzJx2DQy0u -Y4qecEd6aDbqXTo+qOJ7Qm/U+U4kD9MTT6GD +MIICDjCCAXegAwIBAgIJAO2nC4XHXDkUMA0GCSqGSIb3DQEBCwUAMEAxPjA8BgNV +BAMMNVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0 +ZXN0IHN1aXRlMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowQDE+MDwG +A1UEAww1VGVzdCByb290IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9u +IHRlc3Qgc3VpdGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vUDilEsB5 +qI9cGWTthAIjlvr2ngFJqHmMeOgTg4JQQ24MQedh0r22nDNwm80r4RD9RCjlw/k8 +sS+chRwQclJqpE6EV65TIH0JhOKGFpx/Pz/yrru5QwEDkYcHl1QcK3xFUKbSxi/B +MCq4TZf63HkI6/VRY+1SwKF2a4pjWIaDAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADgYEAtBNiRyqydB+iy2DtoYYjsvq/q69o3UrbIhKPMlYE +TJcgWyEz4gsRMnceM/dQl0dchZ8jrAbLiAbqr7PvitjdxGSQJ8w7Gb4IawPu3UCE +TfMWiG5oYV1nHHZotEQuE+Gx4AdzSVGzLGj2xF9dSMxEQq7uPlpv67IeHEn5g3w1 +K5Y= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIB8TCCAVoCAQIwDQYJKoZIhvcNAQEFBQAwQDE+MDwGA1UEAww1VGVzdCByb290 -IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc3VpdGUwHhcN -MTUwMjE2MjAwNjIzWhcNNDIwNzA0MjAwNjIzWjBCMUAwPgYDVQQDDDdUZXN0IENB -IGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3QgY2xpZW50IGNlcnRz -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8wIYcmeePSXVufP/Hn/6ICEog -IUXqSNls5QIJR7Sate4iKGGTDEsRTxI4oDgkOYtcQNuEeXMf6k3xo+PRR08IEQNk -XKy1zUWds6UBFboD72SyuTE2lxJBg+xOAWgl7JSNA+g8e0Y+wfhfxGZgRuqVxVNP -9sAsfCEzGKna1l46dQIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAD20Bmina/uXTLXO -oPWgpMmKwQu7Q6DPXxItCUdWgK1k1D82brRjH+usrkrmCW5BQNXOC/0zJS22ioC1 -CJbhAujH3iPaV0C3xsVSf+bvTL6OMkwV/9x9OdDN+LI2auEt4S+fP5ZTVsTXt4wA -A9cQIl2Qy88enZZAFKxrScFFlstp +MIICCDCCAXGgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBAMT4wPAYDVQQDDDVUZXN0 +IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzdWl0 +ZTAeFw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMEIxQDA+BgNVBAMMN1Rl +c3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBjbGllbnQg +Y2VydHMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMI2MXWSb8TZnCLVNYJ+ +19b4noxRmaR1W2zUxl4aTMfiPt9cK06lNY39EPBfjmb7hjxD76w8fLoV/aZ0gOgd +JXFRZvIg7SyM7QVFma0AJAIZayes+ba1odEmBEi378g0mLrjCLqZtBVHfvJxL/6x +6/flSTAn/+09vtELvvLWBePZAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEAlGC24V2TsiSlo9RIboBZTZqd0raUpKkmVbkwKyqcmecoFfCI +TCmoyJLYyUL5/e3dtn/cGDcaqxaO3qxnstxVEMSrlCGfZdZJ2oouXZMpDy9CkeOM +ypCCx9pc4EmP3mvu64f21+dNCXlhM36pZ1IokeS5jk2FIHUda+m5jlk5o6I= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIB8TCCAVoCAQEwDQYJKoZIhvcNAQEFBQAwQDE+MDwGA1UEAww1VGVzdCByb290 -IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc3VpdGUwHhcN -MTUwMjE2MjAwNjIzWhcNNDIwNzA0MjAwNjIzWjBCMUAwPgYDVQQDDDdUZXN0IENB -IGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNlcnRz -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDDAYtajRx8vM6IB0SLZsAhTD0Y -VHM+/+t0a4m3JXolJBbo9/B2/WAN0IH1E2zmlalLc3JBmGsH1a8U5ZlRow3p2ODL -rFra9FbOl0wekmRFvZeaRln/99dpI5itVpL97QPHO8QMMK1IsyurFA5GfuPOBx9P -i0MvzsT0tYsRvR929QIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAJw4ngOYElfyMYkS -K6bOgMosrBoX8ns6jQgdXEzf7QOIa110bs6nD+XeJeKmzUAZ3wumXBTalPaiqkEz -bq4nlsEs1phvj0Coy5eehjV3DB8bDLEneOlV5N9y4Z4VO1BrhX61bLiPXBRp1MZR -I0sCdxhswSrq02/OuFGe6mqrSBBI +MIICCDCCAXGgAwIBAgIBATANBgkqhkiG9w0BAQUFADBAMT4wPAYDVQQDDDVUZXN0 +IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzdWl0 +ZTAeFw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMEIxQDA+BgNVBAMMN1Rl +c3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzZXJ2ZXIg +Y2VydHMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKpkEBIZexm3YZ94RA+c +vUREqvLgECfHlP9BbkXySFPGWcAPt/0uSW62eVS3UFcB9083W4w/uilL75PXDHV1 +37fyq+6LHCYE5TinzVr5ECAtQMpIzlKkAuAPq3mTa1fklwT/MCz/PKGAljs2o95w +mNyEJwTchOQ52fZjFexRiarNAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEAP1ZhwGxsL7GTNxfs2qwYCjsF2zYSjCPXtwJnKFu5ayGxz6dB +paspokWFCglP1PwPAmINHeqp669WNnAmC5EixdTy2jcnod8NB6RlkOqJmNzVPhvO +cTZXxKd3awOzz0+IJ2bMcC9JPXs8phhRuRgvSfKTTZVtdcFmVF/HYIrBB5Y= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/both-cas-2.crt b/src/test/ssl/ssl/both-cas-2.crt index b0bc3f5b2c..e857f8079b 100644 --- a/src/test/ssl/ssl/both-cas-2.crt +++ b/src/test/ssl/ssl/both-cas-2.crt @@ -1,39 +1,40 @@ -----BEGIN CERTIFICATE----- -MIIB9zCCAWACCQDrgvp38CAy8DANBgkqhkiG9w0BAQsFADBAMT4wPAYDVQQDDDVU -ZXN0IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBz -dWl0ZTAeFw0xNTAyMTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMEAxPjA8BgNVBAMM -NVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0 -IHN1aXRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCyTfGMPAjAylLr3G7c -/QToCA3da5YZzdhd3TiQGugrJjWI4TzVB7pQ8IwDYk/jZf5TzVdEtz0B4TeIeUZl -FLW9dMpa/8SY2TETvMTuXR5MOxyw6FMEKb3buolsIksCCQ1btEIrDZ+gv9SJXcdL -ylU+VI1lKmn2fLNWWATzWrIUawIDAQABMA0GCSqGSIb3DQEBCwUAA4GBAF2T84iG -zWKXu+3PysuPOn7RuRpMgYQKouQktErNJ8hM7Yqj3vu879zUkX1rP0HGnx7xQC3d -nBkoJ7yNDR0MwQpWo1Dj1HLKNEY6ojKJgPd0+m8nG+02yUmmOjo0oMYzJx2DQy0u -Y4qecEd6aDbqXTo+qOJ7Qm/U+U4kD9MTT6GD +MIICDjCCAXegAwIBAgIJAO2nC4XHXDkUMA0GCSqGSIb3DQEBCwUAMEAxPjA8BgNV +BAMMNVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0 +ZXN0IHN1aXRlMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowQDE+MDwG +A1UEAww1VGVzdCByb290IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9u +IHRlc3Qgc3VpdGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vUDilEsB5 +qI9cGWTthAIjlvr2ngFJqHmMeOgTg4JQQ24MQedh0r22nDNwm80r4RD9RCjlw/k8 +sS+chRwQclJqpE6EV65TIH0JhOKGFpx/Pz/yrru5QwEDkYcHl1QcK3xFUKbSxi/B +MCq4TZf63HkI6/VRY+1SwKF2a4pjWIaDAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADgYEAtBNiRyqydB+iy2DtoYYjsvq/q69o3UrbIhKPMlYE +TJcgWyEz4gsRMnceM/dQl0dchZ8jrAbLiAbqr7PvitjdxGSQJ8w7Gb4IawPu3UCE +TfMWiG5oYV1nHHZotEQuE+Gx4AdzSVGzLGj2xF9dSMxEQq7uPlpv67IeHEn5g3w1 +K5Y= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIB8TCCAVoCAQEwDQYJKoZIhvcNAQEFBQAwQDE+MDwGA1UEAww1VGVzdCByb290 -IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc3VpdGUwHhcN -MTUwMjE2MjAwNjIzWhcNNDIwNzA0MjAwNjIzWjBCMUAwPgYDVQQDDDdUZXN0IENB -IGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNlcnRz -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDDAYtajRx8vM6IB0SLZsAhTD0Y -VHM+/+t0a4m3JXolJBbo9/B2/WAN0IH1E2zmlalLc3JBmGsH1a8U5ZlRow3p2ODL -rFra9FbOl0wekmRFvZeaRln/99dpI5itVpL97QPHO8QMMK1IsyurFA5GfuPOBx9P -i0MvzsT0tYsRvR929QIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAJw4ngOYElfyMYkS -K6bOgMosrBoX8ns6jQgdXEzf7QOIa110bs6nD+XeJeKmzUAZ3wumXBTalPaiqkEz -bq4nlsEs1phvj0Coy5eehjV3DB8bDLEneOlV5N9y4Z4VO1BrhX61bLiPXBRp1MZR -I0sCdxhswSrq02/OuFGe6mqrSBBI +MIICCDCCAXGgAwIBAgIBATANBgkqhkiG9w0BAQUFADBAMT4wPAYDVQQDDDVUZXN0 +IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzdWl0 +ZTAeFw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMEIxQDA+BgNVBAMMN1Rl +c3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzZXJ2ZXIg +Y2VydHMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKpkEBIZexm3YZ94RA+c +vUREqvLgECfHlP9BbkXySFPGWcAPt/0uSW62eVS3UFcB9083W4w/uilL75PXDHV1 +37fyq+6LHCYE5TinzVr5ECAtQMpIzlKkAuAPq3mTa1fklwT/MCz/PKGAljs2o95w +mNyEJwTchOQ52fZjFexRiarNAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEAP1ZhwGxsL7GTNxfs2qwYCjsF2zYSjCPXtwJnKFu5ayGxz6dB +paspokWFCglP1PwPAmINHeqp669WNnAmC5EixdTy2jcnod8NB6RlkOqJmNzVPhvO +cTZXxKd3awOzz0+IJ2bMcC9JPXs8phhRuRgvSfKTTZVtdcFmVF/HYIrBB5Y= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIB8TCCAVoCAQIwDQYJKoZIhvcNAQEFBQAwQDE+MDwGA1UEAww1VGVzdCByb290 -IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc3VpdGUwHhcN -MTUwMjE2MjAwNjIzWhcNNDIwNzA0MjAwNjIzWjBCMUAwPgYDVQQDDDdUZXN0IENB -IGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3QgY2xpZW50IGNlcnRz -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8wIYcmeePSXVufP/Hn/6ICEog -IUXqSNls5QIJR7Sate4iKGGTDEsRTxI4oDgkOYtcQNuEeXMf6k3xo+PRR08IEQNk -XKy1zUWds6UBFboD72SyuTE2lxJBg+xOAWgl7JSNA+g8e0Y+wfhfxGZgRuqVxVNP -9sAsfCEzGKna1l46dQIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAD20Bmina/uXTLXO -oPWgpMmKwQu7Q6DPXxItCUdWgK1k1D82brRjH+usrkrmCW5BQNXOC/0zJS22ioC1 -CJbhAujH3iPaV0C3xsVSf+bvTL6OMkwV/9x9OdDN+LI2auEt4S+fP5ZTVsTXt4wA -A9cQIl2Qy88enZZAFKxrScFFlstp +MIICCDCCAXGgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBAMT4wPAYDVQQDDDVUZXN0 +IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzdWl0 +ZTAeFw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMEIxQDA+BgNVBAMMN1Rl +c3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBjbGllbnQg +Y2VydHMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMI2MXWSb8TZnCLVNYJ+ +19b4noxRmaR1W2zUxl4aTMfiPt9cK06lNY39EPBfjmb7hjxD76w8fLoV/aZ0gOgd +JXFRZvIg7SyM7QVFma0AJAIZayes+ba1odEmBEi378g0mLrjCLqZtBVHfvJxL/6x +6/flSTAn/+09vtELvvLWBePZAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEAlGC24V2TsiSlo9RIboBZTZqd0raUpKkmVbkwKyqcmecoFfCI +TCmoyJLYyUL5/e3dtn/cGDcaqxaO3qxnstxVEMSrlCGfZdZJ2oouXZMpDy9CkeOM +ypCCx9pc4EmP3mvu64f21+dNCXlhM36pZ1IokeS5jk2FIHUda+m5jlk5o6I= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/client-revoked.crt b/src/test/ssl/ssl/client-revoked.crt index c38229f847..8448e96125 100644 --- a/src/test/ssl/ssl/client-revoked.crt +++ b/src/test/ssl/ssl/client-revoked.crt @@ -1,12 +1,12 @@ -----BEGIN CERTIFICATE----- MIIBxzCCATACAQIwDQYJKoZIhvcNAQEFBQAwQjFAMD4GA1UEAww3VGVzdCBDQSBm b3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0IGNsaWVudCBjZXJ0czAe -Fw0xNTAyMTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMBYxFDASBgNVBAMMC3NzbHRl -c3R1c2VyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDmApiFBFLZi/hgQOMz -iAHXBbY7A5hNMitQZMSTUB+/fLnzofkUjf/7GiRCLmdTCa4w1wvQp5VbrEhIbSGW -sFSam6GuE0IBfSRJA0IouBtxdk8bCY4HDpXsh/6eC9XtV4k9YDp4JlkUNxOVu8Pb -Z86OEQf3Ww/EZP5AfwORXLYgVQIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAEarnPO1 -Rc88mDYZWM8H/I18L0omdib21+lJczkm4sgv2hVp2nR4Wfb51DojYruLxNJ0k/A5 -T0nEZghQDtNQQpMko9e8jn8gmEAs83zQIsVsmosfTYg0Zr2pSkT0ILSfR6BupHFJ -I96I+qcRKc4rotOirgMrcgo/VpUcWnz8VPEo +Fw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMBYxFDASBgNVBAMMC3NzbHRl +c3R1c2VyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDKeycg+E5SBlaTxcus +Fps4yZnGVB78Kt/HQAZcmgiwWZxN0th9SsJVMw2VkwMPPm4o8idEF/PZvbz15DHk +MrNWSOMB8qsXlt3P8VMhKhWG025TbWpfXbNHKKqQqAc55i1SvQJvllrAYKOlpu/K +YQsIK/ZpjeTywcVi19B3aPE82wIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAE3vuzMF +C5Ut6X981NjsuSlDptqDd8aQqO1HK7McEcH7Tjw6DU9ZPqw9Ktpz/wAJc2DvsmBM +QqlM+OtSkEncAdWx/xOzN46oHUNxrR2cXD1N/0HgVHJEUfq8p+oJHYXKVWtsjO7S +2/fZnMMO9Gv6e7eTiYE55R0IZrQENwtmIJ8y -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/client-revoked.key b/src/test/ssl/ssl/client-revoked.key index b272f70cbe..b2321ca4dc 100644 --- a/src/test/ssl/ssl/client-revoked.key +++ b/src/test/ssl/ssl/client-revoked.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXgIBAAKBgQDmApiFBFLZi/hgQOMziAHXBbY7A5hNMitQZMSTUB+/fLnzofkU -jf/7GiRCLmdTCa4w1wvQp5VbrEhIbSGWsFSam6GuE0IBfSRJA0IouBtxdk8bCY4H -DpXsh/6eC9XtV4k9YDp4JlkUNxOVu8PbZ86OEQf3Ww/EZP5AfwORXLYgVQIDAQAB -AoGBAOV1iXqJya1Fuc8sbHyoHk3IYPeWqoW4mwVkwcbElCeP4mJvH/Glh82VUr7D -VEi+y4vlvN+3j4UY5jN6y5ts5bhDam4RjdHzhLT+ddlztoH4LNcgPDokQtPDtfOd -UbbMcM6Pim7+ynBLncAj7dTin4/pVL2tYUIrKWvLhCU2zISxAkEA+CyHJZs49vOs -hx8xEXGStdLq3k9vUk8G4BISicswixDWPQmJ3YN053FAZ+moHjqNpU5dMn7cVIcA -HEW6LLS7IwJBAO1DbyWtCNQJZBKXzvBKxx9nOBb+5ovQZWs92bpvxkATPn4TUbQx -nEe7eOPX+R2szP9+/3ApmZA1gV1mpVKsyicCQQCcUmf6jzCtlUXKgyJES5bPAwFA -cSa84NyCzb9xnlSAdGWOYvC9YC2GD3czPSHRkK5iPt9DjFc6wyKVrHId8OWjAkBh -8Yp6dRnF3jKPclec3mGg1w1SgNtPMDINuTSeP/IJFWigxvzdc/Vdr0hSVh+iXmkp -t5VfCe04mL3UfsEUhfvVAkEA5Y05DCgaT+rOZbl6hMXlIqT5eT+xTatmDZzv6FUJ -eAaYYhja/FrWa5JFXFUpFTamWGMTkfd6zsDS1bI6hwg/5Q== +MIICXQIBAAKBgQDKeycg+E5SBlaTxcusFps4yZnGVB78Kt/HQAZcmgiwWZxN0th9 +SsJVMw2VkwMPPm4o8idEF/PZvbz15DHkMrNWSOMB8qsXlt3P8VMhKhWG025TbWpf +XbNHKKqQqAc55i1SvQJvllrAYKOlpu/KYQsIK/ZpjeTywcVi19B3aPE82wIDAQAB +AoGAKXV79pFBICRyF8HZSTw7vi3xUZ2p1oJE3bxrUQytGMbQbVLtxwHGtsFEV8sJ +RlbHIZUrmxK4eG4UQdjeqlYDSBsV4x/g4sqpgQAoE1J6TQ9D3at6LqnvUa6ObjQW +W/GczCkrQRkRTxwa35PSE2CcgNcoyUXRkF0DHhugPcOVjRkCQQD1jUQYgLoUKDbZ +mZ1odm2adDu4BMFTd3pyQBVZmetlPrLVr8iU6bHJGUWpaFkpv/jbOJfNgijqQ3i6 +vZRhyHgHAkEA0xi6XZITkzmLP7qEQsho9JX02Pqqpd4ZQOp6CZDY6TH45oOBK0Bn ++xsfGENexiqU2EFtIChesScVpzTuvq2XjQJBALB82HTEEPpr7QB5aKmsdRqOcF3T +DSDwvxFe/flop8gdSGxN690cGqxvfaJFXdCkKjlmc7VB2CaIWD3gBMZDUAECQQC8 +oUIXTurTCf6WSdLZ4j93H3CVWxiV8uraCSxX0+kgKBlj0mrf/UNtLQUSJ1FO/snW +nFApBinnXyeILFKSbIgZAkB74ahj70+NmAFfEcgN3FjcHOy81rsvN9tO4rC0/0nt +iYBz1jc4KBZdyQY1rgk2om9m+EC+mlJsxtgFNK3k/kRJ -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/client.crl b/src/test/ssl/ssl/client.crl index cb86d82a76..72ed9e6d61 100644 --- a/src/test/ssl/ssl/client.crl +++ b/src/test/ssl/ssl/client.crl @@ -1,9 +1,9 @@ -----BEGIN X509 CRL----- MIIBHTCBhzANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0IENBIGZvciBQ -b3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3QgY2xpZW50IGNlcnRzFw0xNTAy -MTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMBQwEgIBAhcNMTUwMjE2MjAwNjIzWjAN -BgkqhkiG9w0BAQUFAAOBgQAsrnXoVeyU8vmxPOVQrHvoMXkEvF9dOnSHIQD0ZnAW -pxbj98hCMSIW+DPIXXFebMQ6GIPp4S/w5kVpngY51paT4iztRMlV+YeyuZQuZX9a -EVgpj4t+i6hhtBHk5p9DeknERoAIsl4m2maQ58lT5UyeN4fdz4eNP6y3mQRfSTUn -bQ== +b3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3QgY2xpZW50IGNlcnRzFw0xNjA5 +MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMBQwEgIBAhcNMTYwOTEyMTYzMDAxWjAN +BgkqhkiG9w0BAQUFAAOBgQAyiU3V6ci5JR5oAZjlG7yFBhVO2TFLga5FynwHK5Wd +ML0BA/0TtTXFiPoul+zvOdqwpX8GC3IuxqgJzlxWOxl5mZzyKEtheT9RBwvBmjAe +ZjT7bFttKo/WKpztNE/2ZEDYyN87Xlpcm5UBFNhcYUjQkxuWIEvH4VOPm0iFjzm4 +tA== -----END X509 CRL----- diff --git a/src/test/ssl/ssl/client.crt b/src/test/ssl/ssl/client.crt index 0c397c02cf..33d57ce0ba 100644 --- a/src/test/ssl/ssl/client.crt +++ b/src/test/ssl/ssl/client.crt @@ -1,12 +1,12 @@ -----BEGIN CERTIFICATE----- MIIBxzCCATACAQEwDQYJKoZIhvcNAQEFBQAwQjFAMD4GA1UEAww3VGVzdCBDQSBm b3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0IGNsaWVudCBjZXJ0czAe -Fw0xNTAyMTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMBYxFDASBgNVBAMMC3NzbHRl -c3R1c2VyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDay1hT3/0Nj5ndv2TJ -DxJZWQTOZD+/0ktqW9qyiRY9o0nf7PCQZE9OCh3ylaaPPfpL7iITZi1KASmSIn7M -E4w1ibmBqFiogDE0Bq0DgJaoeUgLHMERDUtcxBJgwyCGjfI9Om4jy74kwMXb8I5i -jVwZLUTSWzRSgany3WRqMb6CwwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALfP/i7o -ZVYsIZWksIb/uxr/AlghyNQjLPVJTAOjrm9PP9rwKR2alI/zjkDrHVH57n4MfcmD -Xn247DRv/MJFJ1xWCSh4PCy0vyTCFAerNDcqniSTqp2+Yusdr0mH/gHa+34ASYu/ -MXXB4UBMjTnZ/KhaVTmAv3cPeiMAQODRud65 +Fw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMBYxFDASBgNVBAMMC3NzbHRl +c3R1c2VyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDN3RFl8VWMEBN1Qas0 +w1CFcXdDEbKVNSPsqWHzHIEPoGJv+eUIBK2lQ/Ce8nRCdelO50RsmlbcXBIrjVl6 +BN0RmEeEVclgCdiamYN53LBdc5KWKpKCKn45lCtlZodWt0hNNx1pAmh85jDKpoO9 +ErbCnSU1wODPqnOzdkLU7jBu5QIDAQABMA0GCSqGSIb3DQEBBQUAA4GBABUz+vnu +dD1Q1N/Ezs5DzJeQDtiJb9PNzBHAUPQoXeLvuITcDdyYWc18Yi4fX7gwyD42q2iu +1I0hmm2bNJfujsGbvGYFLuQ4hC2ucAAj2Gm681GhhaNYtfsfHYm9R8GRZFvp40oj +qXpkDkYsPdyVxUyoxJ+M0Ub5VC/k1pQNtIaq -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/client.key b/src/test/ssl/ssl/client.key index 6cb6655bc3..4400c8ede5 100644 --- a/src/test/ssl/ssl/client.key +++ b/src/test/ssl/ssl/client.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQDay1hT3/0Nj5ndv2TJDxJZWQTOZD+/0ktqW9qyiRY9o0nf7PCQ -ZE9OCh3ylaaPPfpL7iITZi1KASmSIn7ME4w1ibmBqFiogDE0Bq0DgJaoeUgLHMER -DUtcxBJgwyCGjfI9Om4jy74kwMXb8I5ijVwZLUTSWzRSgany3WRqMb6CwwIDAQAB -AoGAJAMiR7Pvb+L5/XC6QwmzCHfJfbssbwNLHHd/+LDtszmEOFJEik+oafzqTvpo -ztzxrLvGahEAVVT5pa791dNF2V//AKCDj3mOSVfrh6aYeA5naMT91JjnuRVgpdlc -1b7p1FpbnwmzppqSbAfVQxmTlFxvVevukTqkAzP03uuQZ+kCQQD8XMpgCYXFuAl9 -n59OjS9Fi4oISI2lxFFxUK4KjGW4fOzS9/PdHepc4YBJQXSrDdELkH/un5AZQ7tr -67R5YkB1AkEA3fKwaV0dPlXg78rVImUEXwNRM9SgxHquE6itzuT7RYg47bEnDHDm -EGzN5QVs7TrxApk8KCxPUzlv/3vSWszPVwJBAMN+2mN1XQTi4a9IhW+jnZghVce+ -9MQShgjjOEABrRcy538zB94mO5TCN9AH/eo45NUxlnlzcHyx5LHgwUk7HLUCQQCP -RhT/Ty6LiOCVqvf/Jfq2YuvOa5oEe7VX13Grt0FFV3R4a/1rGI5LWBFpoCD62yut -o8GjpUbn0JIt+H6IQuItAkAqnLiP2ZJMhDoey8+btgwiAEoUnkDfa3bNA9wncJhO -M3G4BM/bZhX5cuIaU7kPUHUS5fQeLLUfveWtbVSu1cRn +MIICXAIBAAKBgQDN3RFl8VWMEBN1Qas0w1CFcXdDEbKVNSPsqWHzHIEPoGJv+eUI +BK2lQ/Ce8nRCdelO50RsmlbcXBIrjVl6BN0RmEeEVclgCdiamYN53LBdc5KWKpKC +Kn45lCtlZodWt0hNNx1pAmh85jDKpoO9ErbCnSU1wODPqnOzdkLU7jBu5QIDAQAB +AoGBAMFEdSwGuTCoewwPXcNIRpUxJC1ENStdW1+42atarFPWV/QWYI35jmhkc0dW +Cg3HEwUvm452C2wPyEM5DbK/VCacUkcvcA1taPnNjaw4qUjxRnsVCMhIZp0sCKEW +N1LZhBcc9ThGyOvirRZfk3URqtW58nDqTKZeKZQr/d4DkNz1AkEA7QrHDZUdtdQw +znG+8gsby7hK6+4h3F7ICD+RfUVAHEdSC2L59YsEH03d/kr62t1YOxdlMmCYK9sO +7bthNibwhwJBAN5T8BDD1eRukPPGu602uAPRCfOgx6uoUGL78jTXUYGOiVG/fxkt +EIr3m4G7KKj7LKipX8eowsCVC+Fj/3+SXDMCQAnzPN3OF5wtVwsjbS991eHcT5DN +wzAb7muiN3o5sPI+8Cu4MOPkvPyPaTUmcpdDWVPJrJ7LvTeCD4NdLTx3r/sCQEsV +g+zVhoX4BUIe6sELyseXMEo0EVrapBNZzSmlUiRz89JE3vKssnqMNttwTsIK2cE4 +Ol2ek+8gJvv+nooB7tsCQCu8ZYH75hVZGONfviwHk1RD5DegNZ6pT1Or4g9N23cj +YbP58Lvi4tiQqG6zKMCosWFoDsiKKIH9qQkrygSCn3o= -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/client_ca.crt b/src/test/ssl/ssl/client_ca.crt index 003baed56c..6507bc5f07 100644 --- a/src/test/ssl/ssl/client_ca.crt +++ b/src/test/ssl/ssl/client_ca.crt @@ -1,13 +1,13 @@ -----BEGIN CERTIFICATE----- -MIIB8TCCAVoCAQIwDQYJKoZIhvcNAQEFBQAwQDE+MDwGA1UEAww1VGVzdCByb290 -IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc3VpdGUwHhcN -MTUwMjE2MjAwNjIzWhcNNDIwNzA0MjAwNjIzWjBCMUAwPgYDVQQDDDdUZXN0IENB -IGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3QgY2xpZW50IGNlcnRz -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8wIYcmeePSXVufP/Hn/6ICEog -IUXqSNls5QIJR7Sate4iKGGTDEsRTxI4oDgkOYtcQNuEeXMf6k3xo+PRR08IEQNk -XKy1zUWds6UBFboD72SyuTE2lxJBg+xOAWgl7JSNA+g8e0Y+wfhfxGZgRuqVxVNP -9sAsfCEzGKna1l46dQIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAD20Bmina/uXTLXO -oPWgpMmKwQu7Q6DPXxItCUdWgK1k1D82brRjH+usrkrmCW5BQNXOC/0zJS22ioC1 -CJbhAujH3iPaV0C3xsVSf+bvTL6OMkwV/9x9OdDN+LI2auEt4S+fP5ZTVsTXt4wA -A9cQIl2Qy88enZZAFKxrScFFlstp +MIICCDCCAXGgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBAMT4wPAYDVQQDDDVUZXN0 +IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzdWl0 +ZTAeFw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMEIxQDA+BgNVBAMMN1Rl +c3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBjbGllbnQg +Y2VydHMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMI2MXWSb8TZnCLVNYJ+ +19b4noxRmaR1W2zUxl4aTMfiPt9cK06lNY39EPBfjmb7hjxD76w8fLoV/aZ0gOgd +JXFRZvIg7SyM7QVFma0AJAIZayes+ba1odEmBEi378g0mLrjCLqZtBVHfvJxL/6x +6/flSTAn/+09vtELvvLWBePZAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEAlGC24V2TsiSlo9RIboBZTZqd0raUpKkmVbkwKyqcmecoFfCI +TCmoyJLYyUL5/e3dtn/cGDcaqxaO3qxnstxVEMSrlCGfZdZJ2oouXZMpDy9CkeOM +ypCCx9pc4EmP3mvu64f21+dNCXlhM36pZ1IokeS5jk2FIHUda+m5jlk5o6I= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/client_ca.key b/src/test/ssl/ssl/client_ca.key index 1d636ec1b9..0e6c4649cf 100644 --- a/src/test/ssl/ssl/client_ca.key +++ b/src/test/ssl/ssl/client_ca.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQC8wIYcmeePSXVufP/Hn/6ICEogIUXqSNls5QIJR7Sate4iKGGT -DEsRTxI4oDgkOYtcQNuEeXMf6k3xo+PRR08IEQNkXKy1zUWds6UBFboD72SyuTE2 -lxJBg+xOAWgl7JSNA+g8e0Y+wfhfxGZgRuqVxVNP9sAsfCEzGKna1l46dQIDAQAB -AoGAMAXDmU9G7NvBtuSypvV76txBD7+nbB4ww1XYmMfXmW0kMyiW+rSr/LFjb2jE -H+NMI6KUtzW3Jq2UOyB5e+tqnbDjqZlQjnBYFnWKRa8SxsuamSAvGNLPIzur8rxm -qxOWkxlHpS+I6OXn263sWzG38ptQ3X4zK6ADcTpg7FkkYJkCQQDhCJH630aXQyia -90QM+BaKp7rhr+DHW+vVU/5pg3FrPIuvRlLo+E8iJItY3Ae+AJylK6kP6/5AJzOz -s1tXjZezAkEA1rnW4YIlWRlaJE4hXMNvF4VJO5MBtS60F4/z1kR0uVO5+JAgTZT0 -GE7ghZQ3VwdyRiWc59zXr2qkA75qtMFRNwJBAK0x82iqP6Jbxfy/Ilj4+CBvR547 -xzyourHNm5mJ2Nk4GCombdlwgzc7+SPC9RJ/VhCpsczXTTAC+//qovqXt5ECQEtF -rlwzQWBwkLb1ZKCeKg12vetSZ2DaVGuGHRZZvQlSnnjSHWDU/JSg4fgxswyhIaAR -g2WMd1eY7JIbaFChDBUCQQC46CikUDq2kfIPOkj/dsa4wLkUETrcgBx+eaZbOCgx -JU7GqsoSXxTgKcjZPm/5O/rWWtwB9XhtTuvS/NYi3aSs +MIICXQIBAAKBgQDCNjF1km/E2Zwi1TWCftfW+J6MUZmkdVts1MZeGkzH4j7fXCtO +pTWN/RDwX45m+4Y8Q++sPHy6Ff2mdIDoHSVxUWbyIO0sjO0FRZmtACQCGWsnrPm2 +taHRJgRIt+/INJi64wi6mbQVR37ycS/+sev35UkwJ//tPb7RC77y1gXj2QIDAQAB +AoGALo7NVpEvaDJ+wr74H/uGhMt/PsZFHe7gZvuvPlnxtC1hwywWWbkzWIGlcOqH +edqseIAU0eaCRB4He8MMMBjko5WZcPRrE6mR0ZqtcTSIsg2dRkXJeSbY0A8ZPLjU +xw0RiNPRwcr0zgImzMCR5dVuKOgnAGDRZiDwWefF0g6pRYECQQDwXJyT/E5EKOBY +U4tioFMVypbYlyLXjvhKIFL6wdNAVIOa0LQ+X6cPBZRIM6q+eUjodHWnjf9uFX1i +4mjegCwJAkEAztjruKRIoHAk6zQtSEv2vJhObeXg0gAHWRuCmivS/9NtqrEyGGpl +V0YCe3T257Mrw7A0TgBf7lojkrSnOT+NUQJBAM4Fs7gstTE7EEDlKz4YSd8NzQpN +UXIOe8eduUJyTI6BYmSaq0QjXOBFWfohPyMQdmu5FvfNgLls9hKCGn1Mw3ECQCMQ +tvU4NG+uUzPkRoDpD8zs7O7Id5JiGtzKQxurrjtcNk0neNyWvNNMtQME0w54W0Tz +TAqlGZ4ofbtTEL4tveECQQCFl7OS+Emv0kvUCUm4QQ/xR9bjZ80lRdRn0AwXPiPz +zzYjV0OILDlMip+WrleC99v6R2M6BJrSPQr08oxeIUzy -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/root+client.crl b/src/test/ssl/ssl/root+client.crl index 017f16b6ea..e90d4b9193 100644 --- a/src/test/ssl/ssl/root+client.crl +++ b/src/test/ssl/ssl/root+client.crl @@ -1,17 +1,17 @@ -----BEGIN X509 CRL----- MIIBBDBvMA0GCSqGSIb3DQEBBQUAMEAxPjA8BgNVBAMMNVRlc3Qgcm9vdCBDQSBm -b3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0IHN1aXRlFw0xNTAyMTYy -MDA2MjNaFw00MjA3MDQyMDA2MjNaMA0GCSqGSIb3DQEBBQUAA4GBACEwQiR8BKoD -eGuJKMy73AGLzNu3m7jUPWBPntGpNrUMZXNQXgtfm1t3twXbklQq4pao+9SKgT5X -guXpfoa/mPLs//gsTEx0EQV/YzsXm2xFBUtaRq46GbJK3XTfRJLw7OOzBFij1o3i -GaeVMn7IXwQBNkxQT0AAAiCUz5yz/Wvx +b3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0IHN1aXRlFw0xNjA5MTIx +NjMwMDFaFw00NDAxMjkxNjMwMDFaMA0GCSqGSIb3DQEBBQUAA4GBAAX612LU7WpG +0AsQy1TPAXCwQdvzCVLU2L58unheWGSZruzlaLrh/x435xJ/3a2p9ZxCPlAMTUNk ++4mz4BC14uB7nkKQlTGHH3o1qGhwFTmdXmeDDjzyBPEQkSEfcpLDampRBLaFjq/F +yaKadxocukmCcbxXdiDZePFpf7TfuoIm -----END X509 CRL----- -----BEGIN X509 CRL----- MIIBHTCBhzANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0IENBIGZvciBQ -b3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3QgY2xpZW50IGNlcnRzFw0xNTAy -MTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMBQwEgIBAhcNMTUwMjE2MjAwNjIzWjAN -BgkqhkiG9w0BAQUFAAOBgQAsrnXoVeyU8vmxPOVQrHvoMXkEvF9dOnSHIQD0ZnAW -pxbj98hCMSIW+DPIXXFebMQ6GIPp4S/w5kVpngY51paT4iztRMlV+YeyuZQuZX9a -EVgpj4t+i6hhtBHk5p9DeknERoAIsl4m2maQ58lT5UyeN4fdz4eNP6y3mQRfSTUn -bQ== +b3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3QgY2xpZW50IGNlcnRzFw0xNjA5 +MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMBQwEgIBAhcNMTYwOTEyMTYzMDAxWjAN +BgkqhkiG9w0BAQUFAAOBgQAyiU3V6ci5JR5oAZjlG7yFBhVO2TFLga5FynwHK5Wd +ML0BA/0TtTXFiPoul+zvOdqwpX8GC3IuxqgJzlxWOxl5mZzyKEtheT9RBwvBmjAe +ZjT7bFttKo/WKpztNE/2ZEDYyN87Xlpcm5UBFNhcYUjQkxuWIEvH4VOPm0iFjzm4 +tA== -----END X509 CRL----- diff --git a/src/test/ssl/ssl/root+client_ca.crt b/src/test/ssl/ssl/root+client_ca.crt index 227ab7257a..35dfac2828 100644 --- a/src/test/ssl/ssl/root+client_ca.crt +++ b/src/test/ssl/ssl/root+client_ca.crt @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIB9zCCAWACCQDrgvp38CAy8DANBgkqhkiG9w0BAQsFADBAMT4wPAYDVQQDDDVU -ZXN0IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBz -dWl0ZTAeFw0xNTAyMTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMEAxPjA8BgNVBAMM -NVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0 -IHN1aXRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCyTfGMPAjAylLr3G7c -/QToCA3da5YZzdhd3TiQGugrJjWI4TzVB7pQ8IwDYk/jZf5TzVdEtz0B4TeIeUZl -FLW9dMpa/8SY2TETvMTuXR5MOxyw6FMEKb3buolsIksCCQ1btEIrDZ+gv9SJXcdL -ylU+VI1lKmn2fLNWWATzWrIUawIDAQABMA0GCSqGSIb3DQEBCwUAA4GBAF2T84iG -zWKXu+3PysuPOn7RuRpMgYQKouQktErNJ8hM7Yqj3vu879zUkX1rP0HGnx7xQC3d -nBkoJ7yNDR0MwQpWo1Dj1HLKNEY6ojKJgPd0+m8nG+02yUmmOjo0oMYzJx2DQy0u -Y4qecEd6aDbqXTo+qOJ7Qm/U+U4kD9MTT6GD +MIICDjCCAXegAwIBAgIJAO2nC4XHXDkUMA0GCSqGSIb3DQEBCwUAMEAxPjA8BgNV +BAMMNVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0 +ZXN0IHN1aXRlMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowQDE+MDwG +A1UEAww1VGVzdCByb290IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9u +IHRlc3Qgc3VpdGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vUDilEsB5 +qI9cGWTthAIjlvr2ngFJqHmMeOgTg4JQQ24MQedh0r22nDNwm80r4RD9RCjlw/k8 +sS+chRwQclJqpE6EV65TIH0JhOKGFpx/Pz/yrru5QwEDkYcHl1QcK3xFUKbSxi/B +MCq4TZf63HkI6/VRY+1SwKF2a4pjWIaDAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADgYEAtBNiRyqydB+iy2DtoYYjsvq/q69o3UrbIhKPMlYE +TJcgWyEz4gsRMnceM/dQl0dchZ8jrAbLiAbqr7PvitjdxGSQJ8w7Gb4IawPu3UCE +TfMWiG5oYV1nHHZotEQuE+Gx4AdzSVGzLGj2xF9dSMxEQq7uPlpv67IeHEn5g3w1 +K5Y= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIB8TCCAVoCAQIwDQYJKoZIhvcNAQEFBQAwQDE+MDwGA1UEAww1VGVzdCByb290 -IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc3VpdGUwHhcN -MTUwMjE2MjAwNjIzWhcNNDIwNzA0MjAwNjIzWjBCMUAwPgYDVQQDDDdUZXN0IENB -IGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3QgY2xpZW50IGNlcnRz -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8wIYcmeePSXVufP/Hn/6ICEog -IUXqSNls5QIJR7Sate4iKGGTDEsRTxI4oDgkOYtcQNuEeXMf6k3xo+PRR08IEQNk -XKy1zUWds6UBFboD72SyuTE2lxJBg+xOAWgl7JSNA+g8e0Y+wfhfxGZgRuqVxVNP -9sAsfCEzGKna1l46dQIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAD20Bmina/uXTLXO -oPWgpMmKwQu7Q6DPXxItCUdWgK1k1D82brRjH+usrkrmCW5BQNXOC/0zJS22ioC1 -CJbhAujH3iPaV0C3xsVSf+bvTL6OMkwV/9x9OdDN+LI2auEt4S+fP5ZTVsTXt4wA -A9cQIl2Qy88enZZAFKxrScFFlstp +MIICCDCCAXGgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBAMT4wPAYDVQQDDDVUZXN0 +IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzdWl0 +ZTAeFw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMEIxQDA+BgNVBAMMN1Rl +c3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBjbGllbnQg +Y2VydHMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMI2MXWSb8TZnCLVNYJ+ +19b4noxRmaR1W2zUxl4aTMfiPt9cK06lNY39EPBfjmb7hjxD76w8fLoV/aZ0gOgd +JXFRZvIg7SyM7QVFma0AJAIZayes+ba1odEmBEi378g0mLrjCLqZtBVHfvJxL/6x +6/flSTAn/+09vtELvvLWBePZAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEAlGC24V2TsiSlo9RIboBZTZqd0raUpKkmVbkwKyqcmecoFfCI +TCmoyJLYyUL5/e3dtn/cGDcaqxaO3qxnstxVEMSrlCGfZdZJ2oouXZMpDy9CkeOM +ypCCx9pc4EmP3mvu64f21+dNCXlhM36pZ1IokeS5jk2FIHUda+m5jlk5o6I= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/root+server.crl b/src/test/ssl/ssl/root+server.crl index ac31888a93..096b48ccd1 100644 --- a/src/test/ssl/ssl/root+server.crl +++ b/src/test/ssl/ssl/root+server.crl @@ -1,17 +1,17 @@ -----BEGIN X509 CRL----- MIIBBDBvMA0GCSqGSIb3DQEBBQUAMEAxPjA8BgNVBAMMNVRlc3Qgcm9vdCBDQSBm -b3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0IHN1aXRlFw0xNTAyMTYy -MDA2MjNaFw00MjA3MDQyMDA2MjNaMA0GCSqGSIb3DQEBBQUAA4GBACEwQiR8BKoD -eGuJKMy73AGLzNu3m7jUPWBPntGpNrUMZXNQXgtfm1t3twXbklQq4pao+9SKgT5X -guXpfoa/mPLs//gsTEx0EQV/YzsXm2xFBUtaRq46GbJK3XTfRJLw7OOzBFij1o3i -GaeVMn7IXwQBNkxQT0AAAiCUz5yz/Wvx +b3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0IHN1aXRlFw0xNjA5MTIx +NjMwMDFaFw00NDAxMjkxNjMwMDFaMA0GCSqGSIb3DQEBBQUAA4GBAAX612LU7WpG +0AsQy1TPAXCwQdvzCVLU2L58unheWGSZruzlaLrh/x435xJ/3a2p9ZxCPlAMTUNk ++4mz4BC14uB7nkKQlTGHH3o1qGhwFTmdXmeDDjzyBPEQkSEfcpLDampRBLaFjq/F +yaKadxocukmCcbxXdiDZePFpf7TfuoIm -----END X509 CRL----- -----BEGIN X509 CRL----- MIIBHTCBhzANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0IENBIGZvciBQ -b3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNlcnRzFw0xNTAy -MTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMBQwEgIBBhcNMTUwMjE2MjAwNjIzWjAN -BgkqhkiG9w0BAQUFAAOBgQB1c54zLMueMtLiSmBT6kfXJe9o3Krd2n774g7kzNlR -DeLpCHeUvyLF0m8YK09vbLv2W0r6VQnbjyQGr9xyweRLLtOXc0FIDsTO8g/jvMSq -Q9zITuqWiCHRbNhi2B3HPo2NsrfA+tQEAZvMUgnynlerNvGkLWQZeC2UsxrrSs4t -9Q== +b3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNlcnRzFw0xNjA5 +MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMBQwEgIBBhcNMTYwOTEyMTYzMDAxWjAN +BgkqhkiG9w0BAQUFAAOBgQAm5J6912hKDUWXyu3yCEk1j3KICE2J42ZjFRvxBNdO +Zhv/iBjyFI6TmCVJqoe4GJbNG78xmNEl3/2ZUavG/aD0Z3xGu2xm0p+3Uh2zhfDQ +VEdlgFNKNItS0AtKvoduoZUXKnz3Ft09yLmz9yHLu6EslIsYryi+wnZ5DwUBj5Ec +WA== -----END X509 CRL----- diff --git a/src/test/ssl/ssl/root+server_ca.crt b/src/test/ssl/ssl/root+server_ca.crt index 4a33f77ef6..8b548b46a5 100644 --- a/src/test/ssl/ssl/root+server_ca.crt +++ b/src/test/ssl/ssl/root+server_ca.crt @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIB9zCCAWACCQDrgvp38CAy8DANBgkqhkiG9w0BAQsFADBAMT4wPAYDVQQDDDVU -ZXN0IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBz -dWl0ZTAeFw0xNTAyMTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMEAxPjA8BgNVBAMM -NVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0 -IHN1aXRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCyTfGMPAjAylLr3G7c -/QToCA3da5YZzdhd3TiQGugrJjWI4TzVB7pQ8IwDYk/jZf5TzVdEtz0B4TeIeUZl -FLW9dMpa/8SY2TETvMTuXR5MOxyw6FMEKb3buolsIksCCQ1btEIrDZ+gv9SJXcdL -ylU+VI1lKmn2fLNWWATzWrIUawIDAQABMA0GCSqGSIb3DQEBCwUAA4GBAF2T84iG -zWKXu+3PysuPOn7RuRpMgYQKouQktErNJ8hM7Yqj3vu879zUkX1rP0HGnx7xQC3d -nBkoJ7yNDR0MwQpWo1Dj1HLKNEY6ojKJgPd0+m8nG+02yUmmOjo0oMYzJx2DQy0u -Y4qecEd6aDbqXTo+qOJ7Qm/U+U4kD9MTT6GD +MIICDjCCAXegAwIBAgIJAO2nC4XHXDkUMA0GCSqGSIb3DQEBCwUAMEAxPjA8BgNV +BAMMNVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0 +ZXN0IHN1aXRlMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowQDE+MDwG +A1UEAww1VGVzdCByb290IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9u +IHRlc3Qgc3VpdGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vUDilEsB5 +qI9cGWTthAIjlvr2ngFJqHmMeOgTg4JQQ24MQedh0r22nDNwm80r4RD9RCjlw/k8 +sS+chRwQclJqpE6EV65TIH0JhOKGFpx/Pz/yrru5QwEDkYcHl1QcK3xFUKbSxi/B +MCq4TZf63HkI6/VRY+1SwKF2a4pjWIaDAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADgYEAtBNiRyqydB+iy2DtoYYjsvq/q69o3UrbIhKPMlYE +TJcgWyEz4gsRMnceM/dQl0dchZ8jrAbLiAbqr7PvitjdxGSQJ8w7Gb4IawPu3UCE +TfMWiG5oYV1nHHZotEQuE+Gx4AdzSVGzLGj2xF9dSMxEQq7uPlpv67IeHEn5g3w1 +K5Y= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIB8TCCAVoCAQEwDQYJKoZIhvcNAQEFBQAwQDE+MDwGA1UEAww1VGVzdCByb290 -IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc3VpdGUwHhcN -MTUwMjE2MjAwNjIzWhcNNDIwNzA0MjAwNjIzWjBCMUAwPgYDVQQDDDdUZXN0IENB -IGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNlcnRz -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDDAYtajRx8vM6IB0SLZsAhTD0Y -VHM+/+t0a4m3JXolJBbo9/B2/WAN0IH1E2zmlalLc3JBmGsH1a8U5ZlRow3p2ODL -rFra9FbOl0wekmRFvZeaRln/99dpI5itVpL97QPHO8QMMK1IsyurFA5GfuPOBx9P -i0MvzsT0tYsRvR929QIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAJw4ngOYElfyMYkS -K6bOgMosrBoX8ns6jQgdXEzf7QOIa110bs6nD+XeJeKmzUAZ3wumXBTalPaiqkEz -bq4nlsEs1phvj0Coy5eehjV3DB8bDLEneOlV5N9y4Z4VO1BrhX61bLiPXBRp1MZR -I0sCdxhswSrq02/OuFGe6mqrSBBI +MIICCDCCAXGgAwIBAgIBATANBgkqhkiG9w0BAQUFADBAMT4wPAYDVQQDDDVUZXN0 +IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzdWl0 +ZTAeFw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMEIxQDA+BgNVBAMMN1Rl +c3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzZXJ2ZXIg +Y2VydHMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKpkEBIZexm3YZ94RA+c +vUREqvLgECfHlP9BbkXySFPGWcAPt/0uSW62eVS3UFcB9083W4w/uilL75PXDHV1 +37fyq+6LHCYE5TinzVr5ECAtQMpIzlKkAuAPq3mTa1fklwT/MCz/PKGAljs2o95w +mNyEJwTchOQ52fZjFexRiarNAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEAP1ZhwGxsL7GTNxfs2qwYCjsF2zYSjCPXtwJnKFu5ayGxz6dB +paspokWFCglP1PwPAmINHeqp669WNnAmC5EixdTy2jcnod8NB6RlkOqJmNzVPhvO +cTZXxKd3awOzz0+IJ2bMcC9JPXs8phhRuRgvSfKTTZVtdcFmVF/HYIrBB5Y= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/root.crl b/src/test/ssl/ssl/root.crl index 65e470cce1..2158728f85 100644 --- a/src/test/ssl/ssl/root.crl +++ b/src/test/ssl/ssl/root.crl @@ -1,8 +1,8 @@ -----BEGIN X509 CRL----- MIIBBDBvMA0GCSqGSIb3DQEBBQUAMEAxPjA8BgNVBAMMNVRlc3Qgcm9vdCBDQSBm -b3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0IHN1aXRlFw0xNTAyMTYy -MDA2MjNaFw00MjA3MDQyMDA2MjNaMA0GCSqGSIb3DQEBBQUAA4GBACEwQiR8BKoD -eGuJKMy73AGLzNu3m7jUPWBPntGpNrUMZXNQXgtfm1t3twXbklQq4pao+9SKgT5X -guXpfoa/mPLs//gsTEx0EQV/YzsXm2xFBUtaRq46GbJK3XTfRJLw7OOzBFij1o3i -GaeVMn7IXwQBNkxQT0AAAiCUz5yz/Wvx +b3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0IHN1aXRlFw0xNjA5MTIx +NjMwMDFaFw00NDAxMjkxNjMwMDFaMA0GCSqGSIb3DQEBBQUAA4GBAAX612LU7WpG +0AsQy1TPAXCwQdvzCVLU2L58unheWGSZruzlaLrh/x435xJ/3a2p9ZxCPlAMTUNk ++4mz4BC14uB7nkKQlTGHH3o1qGhwFTmdXmeDDjzyBPEQkSEfcpLDampRBLaFjq/F +yaKadxocukmCcbxXdiDZePFpf7TfuoIm -----END X509 CRL----- diff --git a/src/test/ssl/ssl/root_ca.crt b/src/test/ssl/ssl/root_ca.crt index e491d7317c..fab7a68907 100644 --- a/src/test/ssl/ssl/root_ca.crt +++ b/src/test/ssl/ssl/root_ca.crt @@ -1,13 +1,14 @@ -----BEGIN CERTIFICATE----- -MIIB9zCCAWACCQDrgvp38CAy8DANBgkqhkiG9w0BAQsFADBAMT4wPAYDVQQDDDVU -ZXN0IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBz -dWl0ZTAeFw0xNTAyMTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMEAxPjA8BgNVBAMM -NVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0 -IHN1aXRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCyTfGMPAjAylLr3G7c -/QToCA3da5YZzdhd3TiQGugrJjWI4TzVB7pQ8IwDYk/jZf5TzVdEtz0B4TeIeUZl -FLW9dMpa/8SY2TETvMTuXR5MOxyw6FMEKb3buolsIksCCQ1btEIrDZ+gv9SJXcdL -ylU+VI1lKmn2fLNWWATzWrIUawIDAQABMA0GCSqGSIb3DQEBCwUAA4GBAF2T84iG -zWKXu+3PysuPOn7RuRpMgYQKouQktErNJ8hM7Yqj3vu879zUkX1rP0HGnx7xQC3d -nBkoJ7yNDR0MwQpWo1Dj1HLKNEY6ojKJgPd0+m8nG+02yUmmOjo0oMYzJx2DQy0u -Y4qecEd6aDbqXTo+qOJ7Qm/U+U4kD9MTT6GD +MIICDjCCAXegAwIBAgIJAO2nC4XHXDkUMA0GCSqGSIb3DQEBCwUAMEAxPjA8BgNV +BAMMNVRlc3Qgcm9vdCBDQSBmb3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0 +ZXN0IHN1aXRlMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowQDE+MDwG +A1UEAww1VGVzdCByb290IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9u +IHRlc3Qgc3VpdGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vUDilEsB5 +qI9cGWTthAIjlvr2ngFJqHmMeOgTg4JQQ24MQedh0r22nDNwm80r4RD9RCjlw/k8 +sS+chRwQclJqpE6EV65TIH0JhOKGFpx/Pz/yrru5QwEDkYcHl1QcK3xFUKbSxi/B +MCq4TZf63HkI6/VRY+1SwKF2a4pjWIaDAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADgYEAtBNiRyqydB+iy2DtoYYjsvq/q69o3UrbIhKPMlYE +TJcgWyEz4gsRMnceM/dQl0dchZ8jrAbLiAbqr7PvitjdxGSQJ8w7Gb4IawPu3UCE +TfMWiG5oYV1nHHZotEQuE+Gx4AdzSVGzLGj2xF9dSMxEQq7uPlpv67IeHEn5g3w1 +K5Y= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/root_ca.key b/src/test/ssl/ssl/root_ca.key index e5cddee941..b79f4b0ef7 100644 --- a/src/test/ssl/ssl/root_ca.key +++ b/src/test/ssl/ssl/root_ca.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQCyTfGMPAjAylLr3G7c/QToCA3da5YZzdhd3TiQGugrJjWI4TzV -B7pQ8IwDYk/jZf5TzVdEtz0B4TeIeUZlFLW9dMpa/8SY2TETvMTuXR5MOxyw6FME -Kb3buolsIksCCQ1btEIrDZ+gv9SJXcdLylU+VI1lKmn2fLNWWATzWrIUawIDAQAB -AoGAQ8TmMuO6e/QqUiUlKe8tBzfQdUDn+wTG4N4tGnBvn77VCCJ7qYhXY14aCUs7 -i/V/FcDtE1wF3woHvmJBxDd731TILBhuqn3UIWJafoiezlhqwR2uvTfnWh62N15w -xlmGDuPwXMtQCazbcD6I9hgbADBbMmsyym8cuwN+hxU7bKECQQDfzkAN0RNI/m31 -7GVjOvWrd3+brwf19jXtxhzJCRThjyVyCMYfG9ELP/u76aNCMs2otn2K49Vd0s5A -rG6uN4Z7AkEAy/QXExktz0YxaTMHYHafwSkraoygKVUIoxWm2AHhNXdSY1M8AqkL -6VqGpNgcwiEE0QJHG0MFjB0tZAe9/kq+0QJAWqc+htozR5PXko+ImeMd87BZvgPt -45ExUvi2XDAThzHmZwRqy9sGl9n466q9eGj/qOEShRm4KWLkLIor4uGW1QJAbj2h -u1EA0ei/DH3ontt/vojiTtV0POMZqA0sAdYCRUQZ5FY5ObbmGVw1KyUlZkkysUbp -6HJxrSqYPllw+OKuAQJBAN54Aep6BvzI+arJrOm2Un5l27jfPbuKmvJWjus1mU+e -HkaXYUF31/LIN4gNeu0ULbCSKpvk00UaBfjbwvfLmAk= +MIICXQIBAAKBgQDOL1A4pRLAeaiPXBlk7YQCI5b69p4BSah5jHjoE4OCUENuDEHn +YdK9tpwzcJvNK+EQ/UQo5cP5PLEvnIUcEHJSaqROhFeuUyB9CYTihhacfz8/8q67 +uUMBA5GHB5dUHCt8RVCm0sYvwTAquE2X+tx5COv1UWPtUsChdmuKY1iGgwIDAQAB +AoGAE+iNnmqR/PPCStVhvlUQwgQdt+3II+ew1MuzgPUhZZvKZv3X/zd62cagHndp +E86A1NsfkbNd0NsDYM2ELMmJwC8cTKFw2WyB9t3v0GTtVG8e338QdrrTOvawO3F4 +f4tCESvBgY4qmJMuvicMqLey9fAXc8ul+wocRRYx4r1Gc4ECQQDpgATrxdy7vkf0 +KFxO6htUnKB/V5Q56qvlMzXKHSiwnCMKRYPY7NAxLVNVTmH3ACaBFCvg1f7++Yn7 +r5zEEcuLAkEA4g17NFfFZmBz37C9Cu1W6cX0ps0MgI9w38bEYy8LOk0liUGd+Qit +AKpu8KNOb3v5FQ5TL25EaX1VhM78OE9v6QJBAIFHZPIZGY5E2te+pOT4Tut40I/Q +sHukh0meIdDmdgnaWLguJsKq0tX3b2USwcCcr7TVszmHoegPxyq3X0dbRuMCQQDW +7OhyWO1XrGcfjKQAyq4zMMKvARBc/4TbTtoUT3tGYGlK+jdfuw76LhGy/CIsP1wQ +2ADhfN7QyZjQ4BfQ1j5ZAkBggAL3a/4+KjsPesTxWjlufmoL9QG8Bgaj1tWBYDzX +5CQCWYRPVE7aV+Jh1NDHgToQsziZtvRL16l+GivYEnTX -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/server-cn-and-alt-names.crt b/src/test/ssl/ssl/server-cn-and-alt-names.crt index 04f9b58ddc..ef7fd66b7b 100644 --- a/src/test/ssl/ssl/server-cn-and-alt-names.crt +++ b/src/test/ssl/ssl/server-cn-and-alt-names.crt @@ -1,15 +1,15 @@ -----BEGIN CERTIFICATE----- MIICSTCCAbKgAwIBAgIBATANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0 IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNl -cnRzMB4XDTE1MDIxNjIwMDYyM1oXDTQyMDcwNDIwMDYyM1owRjEeMBwGA1UECwwV +cnRzMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowRjEeMBwGA1UECwwV UG9zdGdyZVNRTCB0ZXN0IHN1aXRlMSQwIgYDVQQDDBtjb21tb24tbmFtZS5wZy1z -c2x0ZXN0LnRlc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMH7OtRvW0qP -gYDMInkd0mgKnqhexEUnTf90mGihzd4sw91J0bJBnC/wfLmpP9a1wOwvAma1GSJ2 -1lLFrSC8bXkT+6nIiqXlFK4HqW5w3PktbO1InujFS1PoxXOdlSwdcIzQ+VDk3Kv3 -IVnCq9w8rcchthnSb+3kYx5QjA0Gb1vhAgMBAAGjSzBJMEcGA1UdEQRAMD6CHWRu +c2x0ZXN0LnRlc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBALsIeqZ7pHER +w0nqVgePeSDdr7/fXMNtF/yFk4ZYUXGLuyzterEUaxeSYB5jmNAeY7ANRjbMb5N9 +mvoHHUsz0AzVNFihcSvP5nB9xIAEypMUF7qoXNVgXbG33gFKLbfNWqbuLUqaiWCb ++B7ahLVPTbm16Kwaw0sEMcSALED/lsMfAgMBAAGjSzBJMEcGA1UdEQRAMD6CHWRu czEuYWx0LW5hbWUucGctc3NsdGVzdC50ZXN0gh1kbnMyLmFsdC1uYW1lLnBnLXNz -bHRlc3QudGVzdDANBgkqhkiG9w0BAQUFAAOBgQCBBVEMkprc18bqWcZ8P93JGc1r -lJoSARfIkBuAkJODyQHJ6kp6fq1kuR8seax35VPNXIvBlPqXoS9zvXYVmF/qOJEk -TtW8YAACZywn02dM5CQRS7T9HCcBJeFUHxbGcBCY+AqzbhM+tGii6UnogjvqdKje -ApVvu0m4MsSn+WWQlw== +bHRlc3QudGVzdDANBgkqhkiG9w0BAQUFAAOBgQAcbbxoyusUuTKq+hz7wLiJfEis +UHrq8BwakaOP0zTln8XBT3uvNeumjQQGciqMNsV8QQ0xT3XadO7R9ix5V5IzTxnC +q4s1xKxSJsmVcPf9Ql43ev3S+lRnyw1ws4lfe9hOdKfOopjHpa+D2VW8/iRfhNj2 +PO7iYMyUZKXB0ynKTw== -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/server-cn-and-alt-names.key b/src/test/ssl/ssl/server-cn-and-alt-names.key index 7577e6f2d4..a1ca8f4286 100644 --- a/src/test/ssl/ssl/server-cn-and-alt-names.key +++ b/src/test/ssl/ssl/server-cn-and-alt-names.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQDB+zrUb1tKj4GAzCJ5HdJoCp6oXsRFJ03/dJhooc3eLMPdSdGy -QZwv8Hy5qT/WtcDsLwJmtRkidtZSxa0gvG15E/upyIql5RSuB6lucNz5LWztSJ7o -xUtT6MVznZUsHXCM0PlQ5Nyr9yFZwqvcPK3HIbYZ0m/t5GMeUIwNBm9b4QIDAQAB -AoGAVOp2gWJR81zI0yoJeT2dyt/DPm9lueQP1+EhisQyC61K/IcBHehsx+udneTC -RmqADqQxh+aFHzoobkmMlUUHInIF8gQe/brw6s27BemUSrT2M47BrZINnOKTvhVa -6xnqcD46DkdYE3z4dF2DsZ+uzgw/bO4sksw/yus2C+2tLlUCQQD8dy5+Ivw7AUVW -H5VNR0joFlR8xeJA8FA460+UhNle/oDtqEjq/YDotHdOnd8EePpR24/c3cMVfXj3 -uqTnKyo7AkEAxLJx8D55ZiDQYprL9DWONVuEk5WZJZIgCNRX+hlymf00Hfm67cue -aD0Y8G1DA5vNywNVpUihdm9wDFPz/PSUkwJAevnG4NRDzq4QyyG5RRpLDhoKb3io -e/9S5FbivbJ0e4w22wzU7/opt7BoSRgnUPNo40Sy79/precfbHQy7ROejwJASovu -zsR+sgwhrh1Iywc5HFPRDTYXUrvs1CvWI/1dB6uFAw9QnysaoBr3xrdCPK3h8t0S -qo+6Ue6uIp32zJnNbQJBALLb34EY6Au69ztILcUpYgzTE8wmXtBTt4RBQDMIw+F1 -ZBw3e3tZjKmOPJySq5v8jyNF5L3s5gd/GRtPRCTkOfo= +MIICXQIBAAKBgQC7CHqme6RxEcNJ6lYHj3kg3a+/31zDbRf8hZOGWFFxi7ss7Xqx +FGsXkmAeY5jQHmOwDUY2zG+TfZr6Bx1LM9AM1TRYoXErz+ZwfcSABMqTFBe6qFzV +YF2xt94BSi23zVqm7i1Kmolgm/ge2oS1T025teisGsNLBDHEgCxA/5bDHwIDAQAB +AoGAB0Hh+HnNvLFywXh9VBfGHHddrXVOVSrzhlHskob0yhIg9jJU03A2Y5jDcApv +UIwNVDR/p/qwzalPDSqfgV6GURgqzS/If+qKN7aPiTZPwTB1I9zNVLf07EaZjS08 +IppwpVbFnrJww1WP/P5VRZxnkbhZ0ClZWm3Bo/V6Axi5O8kCQQDitdVoEerONTq3 +a3n6lzPm473l0P+gZdqbSeqRO59c0uMvE3aIged6cPQZ7WIAivID7Wh7AP0zHetz +NQOCw1fNAkEA0zJfIDyNtkcuPzm9Eg5gQzdgusZ455Eij3VoSRJTX5fLDaBf/lo+ +z6bsGqmnGK8JVtKkpvSPV0L9R47KDCUCmwJBAKlWmKi7eV+9crY+mUYMWsBDrDxU ++Bue+MK1W3hPyKFVBEzNhORB490ZMbuMDH/LSSqV0kzOWFIuLwhGuPCbaKECQEeU +VvFSFKWm0mHTa+VmwfGGH16uTeQOKKx+mm3JrEBF7igcJuzKIWe3p2YSAfQ3vu6S +TgPX940Xw0gxeQFMuekCQQDHuGhGdj3faCSkW8/dIkAX37/VkOxSvaZGpmDxclZL +g21KLL9Ng6wI56wvMo61TCK5gEPnpmDB8H5l3VULt9Yn -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/server-cn-only.crt b/src/test/ssl/ssl/server-cn-only.crt index edf0bb895e..9891072335 100644 --- a/src/test/ssl/ssl/server-cn-only.crt +++ b/src/test/ssl/ssl/server-cn-only.crt @@ -1,13 +1,13 @@ -----BEGIN CERTIFICATE----- MIIB/DCCAWWgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0 IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNl -cnRzMB4XDTE1MDIxNjIwMDYyM1oXDTQyMDcwNDIwMDYyM1owRjEeMBwGA1UECwwV +cnRzMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowRjEeMBwGA1UECwwV UG9zdGdyZVNRTCB0ZXN0IHN1aXRlMSQwIgYDVQQDDBtjb21tb24tbmFtZS5wZy1z -c2x0ZXN0LnRlc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAOqxI3Umy7P+ -FqPSenj/4SgwJgKMM73Q0tJvaDNXb1ipfAUHbvnKCNUs693YjRdZwTAVXsYq8btC -ja/4L24FLCNktLzQfxmVuueMgi7HuYevxVbhOXhxuy8cbTeC6FZj3F6vU7Obg5rM -L6FNzVljbtx/YA2lM4H/lWafTp0mXnmFAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEA -DLLwuxmM5ReGFO6L95kxK++vFa7oXNBw8JxagzqfnF85N2leNbpQxxsDS9U/Bavu -D0okKJR1ezdWlT0AwJcOtnt/X/qoxVFo35rIEjDZv4rWveiPwe3BeYm2tWLRHgKI -6NrPD+kXXqGFPHobbXBPvE2MrW4p+ojD0DTeO8ZXjj4= +c2x0ZXN0LnRlc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBALkR7DNyRnAE +D7ZxnsfOPQ55QB0nM2onJmVZkG4EeqQJ6GZHJym7pHHwbww+dgXvlNzkv2SOvA+Y +q8TXgYvSiKhZ4N4ReSWWZ71P+RqJXpSrj6K2mVKOw0Rno9kMt0370bQOnkvSQY9B +WxJbxji2ks3oj4wma+1zje3i46IlwoYHAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEA +G2ZMgJnCz3/kDv30Uun3YzVktMW4O1y9AbFR1YrbHM8xvVGFLpp9z2PVYOKwKeND +oS3UjW/wJynAT3xPwY3Zg6GbTqx2Fu8BG9bb73RK2af1IT8sB1Pxj8t4kZr0egaO +m8TIbipkZNVakwG9idiVYjn4CusqYthFsOKW+OHiM3I= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/server-cn-only.key b/src/test/ssl/ssl/server-cn-only.key index 9037a9bb8b..f58af68a83 100644 --- a/src/test/ssl/ssl/server-cn-only.key +++ b/src/test/ssl/ssl/server-cn-only.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXgIBAAKBgQDqsSN1Jsuz/haj0np4/+EoMCYCjDO90NLSb2gzV29YqXwFB275 -ygjVLOvd2I0XWcEwFV7GKvG7Qo2v+C9uBSwjZLS80H8ZlbrnjIIux7mHr8VW4Tl4 -cbsvHG03guhWY9xer1Ozm4OazC+hTc1ZY27cf2ANpTOB/5Vmn06dJl55hQIDAQAB -AoGBAN1Vp9oBd5VNqS5g/y4EK+VJ218FuHpoaZsahEv/Rrx4QsU/aHLdDg11qxBy -/UUrWZ2uWc5Mi+ON9bAiQSDicec0ybYD5+Nn3Yv6v82J4Lr6Nlg6lsMSXxr0tfh7 -1Jh4EZWkIvMilSyo2ft2bP5o/rBCiIKXPzLDOmaoYUurNwPVAkEA+uR8icow3Ig4 -DXatPDIVaCr66cfndBSmbXe9M0eY23ic/8VNqjyuo3CNLOqBupl5teZTv6dTLXY4 -9RD5U3x70wJBAO94OTptH8Mp5aJX5PX6x2eggydTBnSNUyZZp1FquFpE5GRhyd5O -RO7V4f0fcZCyuJcZI9xNvkqLIC8WzyZ8FkcCQQCwJk2d/HxzyZv5L/KPCebnvQ1v -p+/EG1+CCgingUQ8CyHHngJaXMKMc9Ba0ccFeQ3v/WedbuBCUffJcAJtcEALAkA7 -fIn60ZDKUmYQ5fSihiFyxJTP9/fqjBDTvgGqX/BbvDFgHkqfRqIpEkiJMH5ti3f/ -UOdvmoBi1Byyld/vl3ORAkEAzruQTKAG5HeD+XPijO1qtaNXAiyxLr49FkJsm/Yx -sgM/ZMbLmYZwu6JHt3+Tvo1++scUuwrsYCUmTP1+Ca37Uw== +MIICXQIBAAKBgQC5EewzckZwBA+2cZ7Hzj0OeUAdJzNqJyZlWZBuBHqkCehmRycp +u6Rx8G8MPnYF75Tc5L9kjrwPmKvE14GL0oioWeDeEXkllme9T/kaiV6Uq4+itplS +jsNEZ6PZDLdN+9G0Dp5L0kGPQVsSW8Y4tpLN6I+MJmvtc43t4uOiJcKGBwIDAQAB +AoGBAJzj6r86UyhG6SMbcyWjWvNYKtgMEXQeOFiW8u+xcF57375E95hTcHb/AsT4 +dolVr3cLnI0cy6TVIli+8R2dnybVxgdV/NSWLk69HDb+YPh2cEA7TxAb3vSfMdyJ +T4uC6ibyjAaWdmEDYhuhP45ALf8MKYHEdtmpdGVU0TtrcZThAkEA8oERu+OPXEVO +OG6yJh6JKwrGOv5jEVK9G2v9ns0LiJDhnDc/wTv5/zq6GIQlWDViV7dmtYPedOAr +Zk3e4HNUHwJBAMNemFSkwMew7jI0yQKuHLN3/kKQXi70ZGXHr6i6hzlcxgoa68zq +Ayx9/4m3D4ucwzSTQo/84p7PA+JXTGLu0RkCQF07WgIOXtNuob/4bu1Q2BOANO4B +Vz0VvjaIsh0XX9PFP7e7VfuIf3isr1c1ltXu0DxA+m/WnvP4KzdNwN4x+KkCQCrP +mr/Jjnjzu26DBJ0yvBVTsQKzEgBmC24GMObfYOxf+QGT3qH7kZB5V7q8w4pLYrct +ocNdnedA49AAYzu2q1kCQQDR5DJ+L/kM02pV4LLhbQ6U6nhBKDPXY26nob/TAtwq +eODDcDiceMGKThwwnGEjEeO2w4uZEM124v5sJgZtlByD -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/server-multiple-alt-names.crt b/src/test/ssl/ssl/server-multiple-alt-names.crt index d15c911e18..c330d4d422 100644 --- a/src/test/ssl/ssl/server-multiple-alt-names.crt +++ b/src/test/ssl/ssl/server-multiple-alt-names.crt @@ -1,15 +1,15 @@ -----BEGIN CERTIFICATE----- MIICPzCCAaigAwIBAgIBBDANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0 IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNl -cnRzMB4XDTE1MDIxNjIwMDYyM1oXDTQyMDcwNDIwMDYyM1owIDEeMBwGA1UECwwV +cnRzMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowIDEeMBwGA1UECwwV UG9zdGdyZVNRTCB0ZXN0IHN1aXRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB -gQC0Bkarg2DSU/0+vFG5dgmxV38SSC006t69zkFmrkUMIEg0iuj4I44qlOf/6EP4 -++RfDwQpiUNRTQmTNwDmjmc1bsXysIvVPzIDKEgpvqI82T1sLpF14PogoNlAzdpu -CnpnU+QTUS3Ic5dhxK8YHyVtsG5nfF/3u1S15p5UaPGiOwIDAQABo2cwZTBjBgNV +gQC3KEGfUWKDHb5tzwJ58o5GaFwUctjQxOg4Wtf6TvBRnuAd7VYAVFRtdPLnH9k5 +dHDUpMw1bHx4nUmbnphtLJDS8VVowLyjKGAU/uOuQidUk0nCSllHPaE8soBZPV8x +BwG7TQ47GO7Jg4dmcTLF+E4m3YbzHglOmoN5+vrwWSJnLQIDAQABo2cwZTBjBgNV HREEXDBagh1kbnMxLmFsdC1uYW1lLnBnLXNzbHRlc3QudGVzdIIdZG5zMi5hbHQt bmFtZS5wZy1zc2x0ZXN0LnRlc3SCGioud2lsZGNhcmQucGctc3NsdGVzdC50ZXN0 -MA0GCSqGSIb3DQEBBQUAA4GBAASEAOEwDFE4qCPXJPpEzma7+vRqVFedWPXFXoW0 -R3HCGlvYJKwnlgxf41ipWHWmJPHLdg+KVJtlfRQ5U2SIIn7yjr3Wk+apcvWMvDpQ -lkIVTwCmSINnj8GjQqgJsHD6I75edRaMQk3PlurzdBWJp6oz+UWbYvGDRDC4pHWu -nLhZ +MA0GCSqGSIb3DQEBBQUAA4GBAANowuGrcHzwfVLHa1PC4W0obG2it61VaA+OFHwv +OAloZTbbNslSh/RGyrD2ZafRZpZNhjNB3JRIt7bv5Y0j5YP7CQkp2ucD90V580Pe +vuoP+jZ/f5ZIC2ffiG9ofPxQdJEHy63GWHSH668rWQBc12GhHqgwZXNoWRMJxVrj +EZqd -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/server-multiple-alt-names.key b/src/test/ssl/ssl/server-multiple-alt-names.key index 64266e3b81..5969b9b5ce 100644 --- a/src/test/ssl/ssl/server-multiple-alt-names.key +++ b/src/test/ssl/ssl/server-multiple-alt-names.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQC0Bkarg2DSU/0+vFG5dgmxV38SSC006t69zkFmrkUMIEg0iuj4 -I44qlOf/6EP4++RfDwQpiUNRTQmTNwDmjmc1bsXysIvVPzIDKEgpvqI82T1sLpF1 -4PogoNlAzdpuCnpnU+QTUS3Ic5dhxK8YHyVtsG5nfF/3u1S15p5UaPGiOwIDAQAB -AoGAPa3gzKbIp4drPvFatsZAb+hgey0LgBPwmOtv8PRIZ+0vkAD/7PSRovk9u6oi -j84N4pvMe0ayL8rLOwsfXd7wcQTxDPxy+RkkMW7RRFzusPjeTMgS753/l4IqehCX -2SLPBkE9e3/UMRR0vds8T7btgTrv3R8pcgntli7W6RPrmLECQQDgZDjxx9X9O36v -SR29RhMUQdz0PQpHYwhtmBLECmr1Lpecu5Zr0JOaabWvd5Lzx1cV2hmldZFQP/gO -fEdzhsfHAkEAzWIjB0y/NH61U4Bj4fML1dGnMEzO0wm0MVEMKjcmPJUbtktvZ6jD -MedYw5VLcWbjXMAJt70UFjcxxAJPmZXZ7QJBAMKEnwiZX1uCc7OoAmvNj0SEQ/JF -598ybl/y8HGZRlb86NkplKAp04qMEL/nPDCvoUKEKq9QV4PlsDd+bMItGIkCQFml -omCHUVZakE84VWDEs7/K2U0t2YEoVSzJkaPDmr8K3qO9XY1Djp/zuTz1p46COG+9 -qwA2WdQwl1pVH+WMESkCQQC1UPYLBYIDj0JaJokSgBPh71Ui8/iBP0J1cvhvKOsS -LrEO4JUq2HBFVcxb7QahHPC22dWI8HlIJgzlUi9BEJPv +MIICXAIBAAKBgQC3KEGfUWKDHb5tzwJ58o5GaFwUctjQxOg4Wtf6TvBRnuAd7VYA +VFRtdPLnH9k5dHDUpMw1bHx4nUmbnphtLJDS8VVowLyjKGAU/uOuQidUk0nCSllH +PaE8soBZPV8xBwG7TQ47GO7Jg4dmcTLF+E4m3YbzHglOmoN5+vrwWSJnLQIDAQAB +AoGAPUp4Y0MNz0il0ANFFd/oYoFLxwADtCEggLNNsRK3cujSoNEqRWPy+Mn4+wT6 +YTKpC0+2km4iXQ5tcmkpIueV8qAitks7n8Ed7qP9Le4MXePnzgn4lL0qY1zExESX +ibAQy/ThPdRuWvelpAXpEOMZclZsix6ksFaAMdC7o8+KwdUCQQDcrd0/X4N+wlSa +LBUcnTmhFI9gcPcvckZoxISHWV2B/QaDVvGA2gYMNJa3lEcuH07LfQh0rraEzAP3 +AhG8BRPnAkEA1Hj2+qKf7aBqcjyQ5yHdxsyw/wFF3ivH5dW7mShi/C6fo+qj6WSU +JNLPN3nhJ++5IH3DuuCTop1qNwk9jEOJywJAQED3bJ5Y4S2gCIvRUdWNlBMyc/gw +YMY7LgIaPHaOvWx42wETrFrO6/rb73PjDdDb1m//aEn+psfoV6FonIA1/QJBAK1D +c4xRf39k2EkN8NA6wsKx+wgIPrR9GUboc1HjKE0jrBUca8wQs+oPauF/Z0eM6nd/ +d1R2fI4YNhxpUaKHFN8CQEpJAGi5/CfFQWdGuDbJwbXwNA8zKlyGohE/lGfjZx55 +LnL3KE/rQXKRaCmLyk5Ce6oZQkDiN7GVEwEz2wMJlys= -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/server-no-names.crt b/src/test/ssl/ssl/server-no-names.crt index 378050cd94..e1305277fb 100644 --- a/src/test/ssl/ssl/server-no-names.crt +++ b/src/test/ssl/ssl/server-no-names.crt @@ -1,12 +1,12 @@ -----BEGIN CERTIFICATE----- MIIB1jCCAT+gAwIBAgIBBTANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0 IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNl -cnRzMB4XDTE1MDIxNjIwMDYyM1oXDTQyMDcwNDIwMDYyM1owIDEeMBwGA1UECwwV +cnRzMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowIDEeMBwGA1UECwwV UG9zdGdyZVNRTCB0ZXN0IHN1aXRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB -gQDE2mzybnsgbq7owCPC0m+igNt5pBk5hDpzFAhpbAZ6hZ85AzHnLbpCDTH5w0Zm -HeevCRkNcDgjqfoDo4DruXYpI8jH+QwuKvUwTt3GGm6C4lb3SBtfNdzJsk5kRE3o -ziMG/OxtiApxFu14nbCnqMuDs3meykD1jHheK0CsHCKq2wIDAQABMA0GCSqGSIb3 -DQEBBQUAA4GBAFNfiKDTLJ2V7kgIWDEOcyKQY8T4cAzgz6jcpN9CePgATB2Yrb9P -x7kkKW68h9SbEk6qtS4YQZjSXWKUqrjjIW22+DJSQAXMZoyADZTnZOASHjNXIzLE -y6B1RX+c7CjolHHSYkbki3RqKGhTQr1hnwkq3N8Fl9bftT5zFuezwnjD +gQC6VU212xAM+d99liOE5ROUG7qwuHhELgNbqGxUgNu8S/nKBrZSsGzWnqA4//iG +DXTZLHszRctXVrkhq2VXFCmRZLajk8uw4GtGCwb/HdvANrDM3rwiU23yjX3Q5Dvh +vHgkG+0PBf2Ghr+/XEbDkAwB0xi8QhO33F+1uQEH4XJM4QIDAQABMA0GCSqGSIb3 +DQEBBQUAA4GBAFBH5fx/I61acluRFTP5RJ8aymi8ez37+MoQ+Aftj1BXwcGSRe2E +57c0VjWUooJGKy/gs/y9F09JngEhFRz0pUCMAKQMaciEJh17ai+QOuSo6/NsGA50 +dw+w4UrYbxJK1RxhgKIYY3sojJ/6G+VbprPlWtSPSEcukRGjj31XRIfp -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/server-no-names.key b/src/test/ssl/ssl/server-no-names.key index 01a09153ca..da86d1e22e 100644 --- a/src/test/ssl/ssl/server-no-names.key +++ b/src/test/ssl/ssl/server-no-names.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQDE2mzybnsgbq7owCPC0m+igNt5pBk5hDpzFAhpbAZ6hZ85AzHn -LbpCDTH5w0ZmHeevCRkNcDgjqfoDo4DruXYpI8jH+QwuKvUwTt3GGm6C4lb3SBtf -NdzJsk5kRE3oziMG/OxtiApxFu14nbCnqMuDs3meykD1jHheK0CsHCKq2wIDAQAB -AoGATKkLWHXx+TVhZD6/LnWpB83KqtpfAGkgIgShKfzpoPk8goVd/7ttF5/v4GZN -miL3QND4MqWLF0hwls4rvKDjBH7q4zw+AR55pnfwoQMsfqMvAn7wZi5HKTah1xbj -yf2J1N62pNW4ZdFnlcXmAPLVDxKyCYaZqdeqgr4VkLvgIVECQQD05OYFasP/5be1 -wSj7zxd5vPK2/EJ6CFN+gwXXYOZWWR7m90g3CXxMWeH7RPIlrfcPC8o8r6xna2BS -E+BKzTYXAkEAzcfLpwZUHcCPrCipMsoC35FQhNCpecuZjVYx0oGsfiE6gu87ddLX -H3YL7+EEmtPdps4fF/9WK87MSpj1IRFv3QJAJIEOTJZqmvV6GeyuGEL5Y9snbuFR -Y3FkSMJtF3rJOuvT8GfB6vpN/e+UAOl5EubIogSG497n2w6lb/aog13thwJADtel -WcO8F3VHJ5y7L32gnW2GyD2gq7dCuQ4Jg+x0e5h79uu4dzQg7hT+oWuygFRdvWVK -mtmA5qIA3DSSIbN3RQJAd97xYxEPMF2NU+vdsLBxrkdH9tCHrqOlzEVTdhBCJrx/ -L/lJQvtxkpWEiFtQdd5OhAurNZ6iWoIdA7fhNHPCqg== +MIICXQIBAAKBgQC6VU212xAM+d99liOE5ROUG7qwuHhELgNbqGxUgNu8S/nKBrZS +sGzWnqA4//iGDXTZLHszRctXVrkhq2VXFCmRZLajk8uw4GtGCwb/HdvANrDM3rwi +U23yjX3Q5DvhvHgkG+0PBf2Ghr+/XEbDkAwB0xi8QhO33F+1uQEH4XJM4QIDAQAB +AoGBAJrFaDr5rqdYlc6W+wHT1SNctQE8+IiJP7jOeMzoC5yn7t9kG+UrLfxG3gb6 +ds/CNaB+VgcMng35tuTEnPRrhuoWh3d0jWZ/QqMklPMyrLO5s0wEOuW47D/KI4MR +wKoucQW44LrTdQgKsa4ZJbILKMScanY1oQXXjq4tueZaxajBAkEA4w8gO1rby9zQ +fIp2C4Mfi9Pe46c1/bM+AD+9hXRO9oYCE+aoi0ww4/qvE5fMYQluSmSd8Yhbvuzk +nifMF2l5xQJBANIVTdPudQvviCoXXtexl71b4KVdX9EjuWTNTCx64YB2ISg30YNw +xvlPvDQy/EvVj/3wSGAy5M/7ZVpkXCQe5G0CQC/Jgi4bzECWo6Zieb+ohB4opDNj +gMB5VeY1hAyvUuMdhxhrJjPTAEMrAmfsPc56bqTnkjpASZbgQqlqlNCkmUUCQHlV +epTLpWhWWMNOqiVTWbsxBGcdrchhpKLWe4c5FWKXV4Ed8/DBQvodFirjw5mc58QX +cgW1fzesD5aMXjcybGUCQQCZwYJir3OQC+CJCrsvACSPr3SQm28hiuO4P41dC7eT +JWluvXOGmWnZwskW/+6imEe7pGYnY81pKThnsV+CXfN9 -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/server-revoked.crt b/src/test/ssl/ssl/server-revoked.crt index 0197116d41..08a5e01526 100644 --- a/src/test/ssl/ssl/server-revoked.crt +++ b/src/test/ssl/ssl/server-revoked.crt @@ -1,13 +1,13 @@ -----BEGIN CERTIFICATE----- MIIB/DCCAWWgAwIBAgIBBjANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0 IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNl -cnRzMB4XDTE1MDIxNjIwMDYyM1oXDTQyMDcwNDIwMDYyM1owRjEeMBwGA1UECwwV +cnRzMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowRjEeMBwGA1UECwwV UG9zdGdyZVNRTCB0ZXN0IHN1aXRlMSQwIgYDVQQDDBtjb21tb24tbmFtZS5wZy1z -c2x0ZXN0LnRlc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMGFtZgJN+Lt -w1Bu6MmAB6h9IUrSFEVxUrrxwz5RG3UDiBkr8StZCM2hXLdSB9tSjBLIWILmuPCR -ydyf70XFTTO8L0Mc6F38I+4GVthNp8h1VJIrl1wRQIfVqFbbKYKiyCQYITzezVuC -UjHjo6xklmMewdInRrcNbWxNVkWH91zLAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEA -m9bRiYypdOrU/1hCzo6bj3Ly39/zUZp+T5xBkLJQpgVLTU8GSEdP35kc3CWzEu77 -39610RY3X0A5fNTLs74t7w2dCViYPvNu/suu87AVtlioHMkwL3QEOUnWM/l23XUR -mj33SwQfmLOV94cNLVTd8IZ9PIT0ARn/YrS1Prx1zeg= +c2x0ZXN0LnRlc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAOmX1G+61akp +XA8zveTsQOtipWN8UzF0TbtxFO6LndM4RMlMFPrS/18KAbOFcfSjQvw9dfMXfOIk +zuwIYtAPD2qVyWAGRvk0Xl5qEz8aaVBbayUN5uRMByF2vgbOMz4IEywNRTWZRS2x +kAFO4/FCb/LRxD+82yck8zcZoL+SdbZZAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEA +LryjYf+t8X5j4DeeRN0uh3OWUpzFo7FaVcakjDQSgZIdlgA9RofL81yzJZAoSqk1 +zT8jqo0HKOAwunbNJxhIH54gAIpsEE9624IUAoLMo8OUF2WdMWLXJJTljfbXx/Tb +4ccuQvMLs+Um70Ogc7Mqm0+BM5N61lBdJOgMlDP511E= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/server-revoked.key b/src/test/ssl/ssl/server-revoked.key index b12071a52d..33004170b7 100644 --- a/src/test/ssl/ssl/server-revoked.key +++ b/src/test/ssl/ssl/server-revoked.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQDBhbWYCTfi7cNQbujJgAeofSFK0hRFcVK68cM+URt1A4gZK/Er -WQjNoVy3UgfbUowSyFiC5rjwkcncn+9FxU0zvC9DHOhd/CPuBlbYTafIdVSSK5dc -EUCH1ahW2ymCosgkGCE83s1bglIx46OsZJZjHsHSJ0a3DW1sTVZFh/dcywIDAQAB -AoGADWTrxLIepB5AvnhutEOgRBElFMCllojZaQcCtHV4qQititB3yMBI07KvcCDF -WnDEMSict7KwajYs+pA3R2T4itVLCoz5iXYJ2CIf6XOJK+NYcf7XulSm5IqukbqT -3KlofUY2GY/5DN9tgUUnAsZ7wh6iMaq/H+BPBcblZg2kyYECQQDpYRAjwepzpr0P -gfohKYlfKJwQ9WWTRkMasn6q4DY6txeXNk5nMC9C3FHeiTgpfRr8GZBvk61lb6pV -pFWADR2TAkEA1EepQ95Mums8BxU6/PAOhXKLlyYvldaIXcajv/+/PclVuEL8ko5z -jspEGk7U/jqonwcN98R/h4ui7nxhoxIG6QJAFydgGIwWnJ7ARxeYH04lqOE4ip4u -E6x23+Exm/ZeqvibSI9EvAwVxEZjgPaQMd2NndFTeR5np5aqiZCiQvAKLQJAfRs+ -xqDc14Ebf5Ejkq5n4H4BhrMamFQ3Sg0ntKAlNWTTACV6dWU+9Yh/WoHbRXmMpyyh -LsS/5EKHY8YqRND7AQJAd+qIgqFUI0RAwvbmLxW/iR5JIKM5kZ4xJ13/O4x55XEI -4H+8YS/nYPnjMpaEWrFppNfv2UEXD2L1OkJVuYx1Sg== +MIICXAIBAAKBgQDpl9RvutWpKVwPM73k7EDrYqVjfFMxdE27cRTui53TOETJTBT6 +0v9fCgGzhXH0o0L8PXXzF3ziJM7sCGLQDw9qlclgBkb5NF5eahM/GmlQW2slDebk +TAchdr4GzjM+CBMsDUU1mUUtsZABTuPxQm/y0cQ/vNsnJPM3GaC/knW2WQIDAQAB +AoGATDyWQ6TZiK0L85Yyep00jt4SFkcEK9bGa897QmNkrgPmR0BCdJ4aZF0ysvFx +gKMsAIDaluzqgC/9LIGMJlVT9RisKWQks2cPIs5gERmYg7uNzfYegbn4N2liRG5z +d7aWevi82Dtie1xch2DdUW/mxGdvR4duyXOlSYUhbDmK8oECQQD4MylmccYxz8Ry +7APVKXPMhSCmD86y4KGfWGtKuLo4vF4Ifaze7sVtEBznNQLIAn2U4M5XD4to/VQN +2nT1ESHJAkEA8O8nDKC4vWtCXzMD2/DdHrJjxhCPbDJFp24PRFUia1yUOAgqcnfv +UzXkdcxSv/zUBB/WejLEvvrQ6Ib48Lq4EQJAZ8ashsMHhYhDsXFxYM1GN7tqHUT6 +vdwid8e2hLWcV2CbSJ2TjFr1fVaBX0LQ+OPhskAUxl4fgjR50pkG0fjp0QJAJvB/ +/yp6sSKEt54nIYTsN+nc9kX26CW33DeNgB3CUlfEHMo1EgsQQwKSyfcb6KuUGJaM +s3NBGRywZuRpl36WAQJBAKgyfmq15Ggoe4rQOyO774e1mfUQW6/fWVX8dE09ZxXz +28Sy4/K16UDUzIzzuaW3L7WjzCKql82vy2PpmWX28OM= -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/server-single-alt-name.crt b/src/test/ssl/ssl/server-single-alt-name.crt index 349792fc28..832aa051e7 100644 --- a/src/test/ssl/ssl/server-single-alt-name.crt +++ b/src/test/ssl/ssl/server-single-alt-name.crt @@ -1,13 +1,13 @@ -----BEGIN CERTIFICATE----- MIICBjCCAW+gAwIBAgIBAzANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0 IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNl -cnRzMB4XDTE1MDIxNjIwMDYyM1oXDTQyMDcwNDIwMDYyM1owIDEeMBwGA1UECwwV +cnRzMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVowIDEeMBwGA1UECwwV UG9zdGdyZVNRTCB0ZXN0IHN1aXRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB -gQDnDMJShFXba5o4o1ubyRmV9AyJLVM+8nZaC1iJzqeIPObXYpbcp3bhXtowAwvk -d3IGI/fBm/2/NKvYnyagTS9DUNRTnykHxlCKsMitx38+sU1EerkDltK0OId+obvZ -eVD+h3j7pVfA0NPKXkpcP3xoihQU9I5kOPKQEIQPNAUfdwIDAQABoy4wLDAqBgNV +gQD8lRO7m7xRlUJCKdkHdPnLKu/pbHLBgIYJk33nr22CQaM0UCgfCzr0LPaPCtlF +r5D/WS/MIlyzaXHHJMnzbFB1bx3T9BQijobeO1LKS7s3ZyPEaiNVJoih6ZXlXMQ/ +sPp887EChn+COh9BWgnWbSV0Zq2m9bggDg83J34bIeWOmwIDAQABoy4wLDAqBgNV HREEIzAhgh9zaW5nbGUuYWx0LW5hbWUucGctc3NsdGVzdC50ZXN0MA0GCSqGSIb3 -DQEBBQUAA4GBAHDkKwNT8p+/sv0Q2dIo0jdFxtBdS/fnsHzjr5ZJNa9SXEer5m3v -lMllzYW8lfTdR9PgBZ3y2lFZixD63ZF3nkIEYcD5njLp4ui7g2OVqxRfmt+rHefh -HiKm5v5GLs72lhR4GQT13AsjGVS1WWZtYhO4LwTjN+nbjnRIpXIhrSC/ +DQEBBQUAA4GBAIIcaM6MHSbH9t8cQhgWKXhjqlph/oRsNwTTalSamLR6JeT9BWxp +IZXrsgzIJsVlTSHj8JRZnv++6TVDe/1N2MtwNfJUqnblIFwaf83kulv18Vhwoh4l +dqPNaBZqdk9+EWJwPSzolK2VRKZcea+E2sMZBTYAV3pijy1k/oC8OX1V -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/server-single-alt-name.key b/src/test/ssl/ssl/server-single-alt-name.key index 71fd85c0db..90b6bc4663 100644 --- a/src/test/ssl/ssl/server-single-alt-name.key +++ b/src/test/ssl/ssl/server-single-alt-name.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXgIBAAKBgQDnDMJShFXba5o4o1ubyRmV9AyJLVM+8nZaC1iJzqeIPObXYpbc -p3bhXtowAwvkd3IGI/fBm/2/NKvYnyagTS9DUNRTnykHxlCKsMitx38+sU1EerkD -ltK0OId+obvZeVD+h3j7pVfA0NPKXkpcP3xoihQU9I5kOPKQEIQPNAUfdwIDAQAB -AoGBAOP42uPAX1aY3Rp1VLZpvi0PGC9h4XmCkvRVrY6LsRHjxYFPbbtaIRpOFMq6 -tsk+cetNIfCOkdhPiB+9KMeSYMYShiyCrHfFxuS0FIP4rQhBB89wzcjffw2CYLGD -Umx65+XVv6RBW85p6v4s1+LQMVUtf41yxm9JXT0TVDjEcgRBAkEA+/FKxv9DuZNZ -Abjak3MeaULpnPl+Fxp+jg1M4wK12MFYCm2eBUx0X+cqVORErwLJ3gdXQBT7fJQz -bNwxjUKuTQJBAOrFVKF2dtuPAeFBlKG4sy5azGfgzS6cAJQ4LPp4uGX7ve9C8OzI -oZU21LT4cm3nuFSeMjcCKHmur85gFQrETtMCQQDKWu1yk8gzn1OX/H8iew3sAaBd -Qk6yA8euFKSymJSyOeiax5xqKRQ3ixYHBSjdYGH/AOplP/UWBHqhbuIl0W7pAkAr -f9qZfCizr8CqawtOF7njeeFr0eRSoYcd73auBhYsl0NvBJk9VkNSMXGiAnK5WHj3 -/MPTG2xCd5KNi5H6h7sPAkEApf8JUvEA5ZPkFAA6x+OXLmEL+nXOnJnhKjSUIVJx -Pgp7FTy6eKg+/iUEyhRHw5So7QjwHqH61+CIBNS41vGPuA== +MIICXwIBAAKBgQD8lRO7m7xRlUJCKdkHdPnLKu/pbHLBgIYJk33nr22CQaM0UCgf +Czr0LPaPCtlFr5D/WS/MIlyzaXHHJMnzbFB1bx3T9BQijobeO1LKS7s3ZyPEaiNV +Joih6ZXlXMQ/sPp887EChn+COh9BWgnWbSV0Zq2m9bggDg83J34bIeWOmwIDAQAB +AoGBAOkekmraLvJBRzkXtJZcUVxBkdIn5LZRb+SQu2jFkdXhzMawoIceb1gD08Br +6+KUSshSQXov0M1KKdf6TWYc9xfGg0+XgqPLw0CIJjdO8TZkIaaHZU803snWtM0e +9PlhUm2T/RDdx0cG7HD4WR8x5ij1Fc+W5QsDOi5oCke0XC0xAkEA/+rmRUaOSOaI +CohIH+oHlLTLfr0acAP2cjQHJkf5OUBVnyirfCcsFsxmnhvARuuIdftwd9p73Gw9 +MIGVV6TDIwJBAPyp5xByHy7sicsi2ynSTpS9JhuVvR9cdKBLGmbPQtS0Fb0rannR +wyybfFIN1trfZJ6QmNKAPSPfUFhre3b4WikCQQDVOZYaairquojmnZ3aWVdvoyNZ +uZ1pbyPLC2ZZkuYnuV6deXlHvCuT40Iswdp2PJA6HQEcG0HP6a8h1xXjLDgZAkEA +wMKirTJTxgnh6l9SUyrGlsVjoGHx4k44D96catkvBHXLrAHGft/ghlStWTCDvYH3 +Et4AKYB6pLaHZp3BmPdKWQJBAMI65XkJC5+XESMUP26lC71eo6tMcJ9fBKOa7PEW +M9I04AeId/3nbA2eKCebfUzrZOizeHHOjhH3Dubz3df8Ww8= -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/server-ss.crt b/src/test/ssl/ssl/server-ss.crt index d0c9b83923..9b4f4d1e80 100644 --- a/src/test/ssl/ssl/server-ss.crt +++ b/src/test/ssl/ssl/server-ss.crt @@ -1,13 +1,13 @@ -----BEGIN CERTIFICATE----- -MIICCDCCAXGgAwIBAgIJAJyw4sQKTY2UMA0GCSqGSIb3DQEBCwUAMEYxJDAiBgNV +MIICCDCCAXGgAwIBAgIJAJ5i7OAq01pyMA0GCSqGSIb3DQEBCwUAMEYxJDAiBgNV BAMMG2NvbW1vbi1uYW1lLnBnLXNzbHRlc3QudGVzdDEeMBwGA1UECwwVUG9zdGdy -ZVNRTCB0ZXN0IHN1aXRlMB4XDTE1MDIxNjIwMDYyM1oXDTQyMDcwNDIwMDYyM1ow +ZVNRTCB0ZXN0IHN1aXRlMB4XDTE2MDkxMjE2MzAwMVoXDTQ0MDEyOTE2MzAwMVow RjEkMCIGA1UEAwwbY29tbW9uLW5hbWUucGctc3NsdGVzdC50ZXN0MR4wHAYDVQQL DBVQb3N0Z3JlU1FMIHRlc3Qgc3VpdGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ -AoGBAOqxI3Umy7P+FqPSenj/4SgwJgKMM73Q0tJvaDNXb1ipfAUHbvnKCNUs693Y -jRdZwTAVXsYq8btCja/4L24FLCNktLzQfxmVuueMgi7HuYevxVbhOXhxuy8cbTeC -6FZj3F6vU7Obg5rML6FNzVljbtx/YA2lM4H/lWafTp0mXnmFAgMBAAEwDQYJKoZI -hvcNAQELBQADgYEAGweDmEYzoEWb3WNn7Mc58ToPnl5DbRZdVmRjsyC6J5oZRu2E -e/GZZ/1MSNPgccoyhdcPmSqTzUzbQnvYsqcHfuncA/oNJR3wvMV/wSy0QepklX1b -ixjZg9c+mhQ/JTSjYnRK5iSTPNX4F3zkpvP79POuQYl/7Oihqxl0Mmkezuc= +AoGBALkR7DNyRnAED7ZxnsfOPQ55QB0nM2onJmVZkG4EeqQJ6GZHJym7pHHwbww+ +dgXvlNzkv2SOvA+Yq8TXgYvSiKhZ4N4ReSWWZ71P+RqJXpSrj6K2mVKOw0Rno9kM +t0370bQOnkvSQY9BWxJbxji2ks3oj4wma+1zje3i46IlwoYHAgMBAAEwDQYJKoZI +hvcNAQELBQADgYEAZtnJALcQmbqBAm16RE/Smu75lBkniqlB0MwOJyGpDg5DkXtA +YnZP139cRWKCjbWiYj4hgK0eGGRoBuubF6zRrRlYLV9iyZyRx1cBYyUbQBW+AfER +jWiL4IBJWn7RNej6Uc0Th8Di5eEZapDt0DFkdALFhpFNaWJIcCUKpqEJHUA= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/server-ss.key b/src/test/ssl/ssl/server-ss.key index 39cf3e3c05..69bc907b74 100644 --- a/src/test/ssl/ssl/server-ss.key +++ b/src/test/ssl/ssl/server-ss.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQC6FTHDuNKbYQChNtxAFLKzJESIKOZh8WpgCN91HFtnXX4hp3rS -bkEDIABlQdqfcXLk7PmlR/rboOIqwuIAaIa12BxEJ5KW2vtcSAFd17anhG/a9n8w -cQnoEUvLLAp7V2xGJ8Cu8mtyv9Qsmd5bS+SFchmbIcMZUb4znZfIr7AWWQIDAQAB -AoGAJvVzAtA6P8+ySw5qVHxA4aKxOnSdr1nU9KBG8ITsWhrH4pHm9BGjSN01V/3O -oN0mueknZ0RHsB3h3CQLHxzDPwmsah7apU8W/1AVyZ9LDEMuoZQef3+JfegmuNMj -YYtBR8xozTviOH0UH6t3VOW8Y2TLtZo5kMz3XwjWBS+cCYECQQDlPEfH1x9QGXNW -Eo37QK4UkL6/2czIXWitvb5+79KiG70XYIxrQR9NhpZHSGjBlS+TqJ4tnQa/fv95 -v4I7Q5NpAkEAz88ax91FeHr8y41s01MmJ6Gs6EOrFEpoHGboDdbwJ50pME5XnVJu -xjHPklHgwiWFf4dQURjv6hCPUMVpe1w9cQJAZocPk9Ijry+y5kxmNHo5YflbV3OS -pAsjRpIXIa8iBl9hs5L7Ov1lgscvb7JzKCIRpXlFRiF1YzDqEwoUtW0EAQJAH+/c -VcsT2ihMoZvilbe5rW2TfT6pFD07MuI916Ko1e25Xssre+onTB5roDklKbFKiwbo -uQ30ESzqWad9RpAugQJBANmRD25BmlHbdBDg+Zfd+4jDPAjXN8OesslEs5dMvs8C -vqGrozvmtpLRcLiIitTiT4TzuUPowgZQtCjC0X6jSGY= +MIICXgIBAAKBgQC3ekl7kd5dbcgs5UDu2im/oNSranNRqfqe4USVtiD9NPEWIc43 +wJhkp7w5BOf4xwWukrhOkpTzPLJwYH5HsQL6gBFZi095VExrdRopmpk268l41rSD +q+y0eWM+DoapoiU8tZ106RtMVSinDfXpXz6Nh9+WQ7/Q4EHWbREvf/SIeQIDAQAB +AoGAWbacqaRIk2xznag3WNMp6L5OXsa9Pmgb2IYTkBSvCsBRRd4fxFkS6tythz/j +4VwHZjXtktXPqSO7qIE2Hf3qkxfBpZ72qrvEDpHLXzEFXUamJMPRDZIBHrkfa/sl +pq+z0siCwO/ozoiInQFxArHeZs8MoGd/FYtECwbuvQd9LuUCQQDkYhlrELE7MB47 +Ot+vgkodp84p2LHlY48cLfn0Es+3aDzusWZbNDENmY9tOubIygh3qo0G2NtSEGVK +NZzm73GHAkEAzaoUKK3YCD95OKyj3FZ9P/32K1y7JqCH/9ux9dXBQLegdd0hWLk9 +USLlaPgC4FOVrSSbiQHbZ8lUewwtNeK1/wJBALf0nGy0wUzfcTpcLZh85Z4Fb/Yc +6Q3Pp5IXJmIGVPFyMMJCeiO0Yl6F9hURgJrywOdCpN2DBwWO10dy77LD4zkCQQCP +EJHnXk8aJbVYpFd0TcHhAvP8ZAxYKXGRnS0lWqWNNG9trf6lbm5mA4VcSLIPhHVp +NT7wxpbukpGu6uCETInXAkEAhF7m+XeHvJ0vrdpI3OSOVTW8o9QZSqUffsGB6eRb +v/dJvKgaz3JOzpvRS87lRUd62QfjfmiDXjyhllRn8OPMuw== -----END RSA PRIVATE KEY----- diff --git a/src/test/ssl/ssl/server.crl b/src/test/ssl/ssl/server.crl index d36ce7fe75..e481d011f6 100644 --- a/src/test/ssl/ssl/server.crl +++ b/src/test/ssl/ssl/server.crl @@ -1,9 +1,9 @@ -----BEGIN X509 CRL----- MIIBHTCBhzANBgkqhkiG9w0BAQUFADBCMUAwPgYDVQQDDDdUZXN0IENBIGZvciBQ -b3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNlcnRzFw0xNTAy -MTYyMDA2MjNaFw00MjA3MDQyMDA2MjNaMBQwEgIBBhcNMTUwMjE2MjAwNjIzWjAN -BgkqhkiG9w0BAQUFAAOBgQB1c54zLMueMtLiSmBT6kfXJe9o3Krd2n774g7kzNlR -DeLpCHeUvyLF0m8YK09vbLv2W0r6VQnbjyQGr9xyweRLLtOXc0FIDsTO8g/jvMSq -Q9zITuqWiCHRbNhi2B3HPo2NsrfA+tQEAZvMUgnynlerNvGkLWQZeC2UsxrrSs4t -9Q== +b3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNlcnRzFw0xNjA5 +MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMBQwEgIBBhcNMTYwOTEyMTYzMDAxWjAN +BgkqhkiG9w0BAQUFAAOBgQAm5J6912hKDUWXyu3yCEk1j3KICE2J42ZjFRvxBNdO +Zhv/iBjyFI6TmCVJqoe4GJbNG78xmNEl3/2ZUavG/aD0Z3xGu2xm0p+3Uh2zhfDQ +VEdlgFNKNItS0AtKvoduoZUXKnz3Ft09yLmz9yHLu6EslIsYryi+wnZ5DwUBj5Ec +WA== -----END X509 CRL----- diff --git a/src/test/ssl/ssl/server_ca.crt b/src/test/ssl/ssl/server_ca.crt index 517a30aa4b..2bbb8c9c1b 100644 --- a/src/test/ssl/ssl/server_ca.crt +++ b/src/test/ssl/ssl/server_ca.crt @@ -1,13 +1,13 @@ -----BEGIN CERTIFICATE----- -MIIB8TCCAVoCAQEwDQYJKoZIhvcNAQEFBQAwQDE+MDwGA1UEAww1VGVzdCByb290 -IENBIGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc3VpdGUwHhcN -MTUwMjE2MjAwNjIzWhcNNDIwNzA0MjAwNjIzWjBCMUAwPgYDVQQDDDdUZXN0IENB -IGZvciBQb3N0Z3JlU1FMIFNTTCByZWdyZXNzaW9uIHRlc3Qgc2VydmVyIGNlcnRz -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDDAYtajRx8vM6IB0SLZsAhTD0Y -VHM+/+t0a4m3JXolJBbo9/B2/WAN0IH1E2zmlalLc3JBmGsH1a8U5ZlRow3p2ODL -rFra9FbOl0wekmRFvZeaRln/99dpI5itVpL97QPHO8QMMK1IsyurFA5GfuPOBx9P -i0MvzsT0tYsRvR929QIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAJw4ngOYElfyMYkS -K6bOgMosrBoX8ns6jQgdXEzf7QOIa110bs6nD+XeJeKmzUAZ3wumXBTalPaiqkEz -bq4nlsEs1phvj0Coy5eehjV3DB8bDLEneOlV5N9y4Z4VO1BrhX61bLiPXBRp1MZR -I0sCdxhswSrq02/OuFGe6mqrSBBI +MIICCDCCAXGgAwIBAgIBATANBgkqhkiG9w0BAQUFADBAMT4wPAYDVQQDDDVUZXN0 +IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzdWl0 +ZTAeFw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMEIxQDA+BgNVBAMMN1Rl +c3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzZXJ2ZXIg +Y2VydHMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKpkEBIZexm3YZ94RA+c +vUREqvLgECfHlP9BbkXySFPGWcAPt/0uSW62eVS3UFcB9083W4w/uilL75PXDHV1 +37fyq+6LHCYE5TinzVr5ECAtQMpIzlKkAuAPq3mTa1fklwT/MCz/PKGAljs2o95w +mNyEJwTchOQ52fZjFexRiarNAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEAP1ZhwGxsL7GTNxfs2qwYCjsF2zYSjCPXtwJnKFu5ayGxz6dB +paspokWFCglP1PwPAmINHeqp669WNnAmC5EixdTy2jcnod8NB6RlkOqJmNzVPhvO +cTZXxKd3awOzz0+IJ2bMcC9JPXs8phhRuRgvSfKTTZVtdcFmVF/HYIrBB5Y= -----END CERTIFICATE----- diff --git a/src/test/ssl/ssl/server_ca.key b/src/test/ssl/ssl/server_ca.key index ac4e76f2b2..668c37bff9 100644 --- a/src/test/ssl/ssl/server_ca.key +++ b/src/test/ssl/ssl/server_ca.key @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXgIBAAKBgQDDAYtajRx8vM6IB0SLZsAhTD0YVHM+/+t0a4m3JXolJBbo9/B2 -/WAN0IH1E2zmlalLc3JBmGsH1a8U5ZlRow3p2ODLrFra9FbOl0wekmRFvZeaRln/ -99dpI5itVpL97QPHO8QMMK1IsyurFA5GfuPOBx9Pi0MvzsT0tYsRvR929QIDAQAB -AoGAcq9i1INvAJFN6cRUdKOeVTbwK3HnQWLjh9mC6bpZxqQd8S94NZK4Pgelloux -HT9hjGU+CgPo1ne+e0y4ycFaeWf6SFyMJ3KmGFKCliE6A5zd/g+rIp8oja0Y7eLZ -PUdx984qynfvFMxgB+VJk22cLui9az65WCY+akdWbnwfR4ECQQD4GH6S71bZya9G -/DDS2YYi3Cvke6wsGSXTMyfDaW42M3mtJOrmoczrx1sAzTmO4rwhuzFFQRs662IS -/c9nmXOhAkEAyTgK9BNbkb5n2KN0Ebpx+x9cCh7fJ6qY54DOk+svp2jOhBcV9Aqd -fYPHzPI0v358buPjozXgALNl7FGrO6sC1QJBAPKrwuMmiOVuiav9ciRL8RCYG7bZ -4Ycg8garuvFBZzRNFW9u9PWyvibCURlvpCVHUo4L9B2xmVkAdGXvLbhAOQECQQDD -9zKjtl6NuFRGphmaUmxDV605pgtLBFhZzhZh9MC6V9YYyqr0u4nZ/YeOz6wTe0oQ -bRz7jLKVvCHdX0RWnhvpAkEAhY+plw7q6fyXSBBOVUcHUO2Wtmdm8clvKbs64Wdl -bjryhvBhq3gPii7jnLGwS2v5jwqCcKpK1tszO/8+gj2T+A== +MIICXAIBAAKBgQCqZBASGXsZt2GfeEQPnL1ERKry4BAnx5T/QW5F8khTxlnAD7f9 +LklutnlUt1BXAfdPN1uMP7opS++T1wx1dd+38qvuixwmBOU4p81a+RAgLUDKSM5S +pALgD6t5k2tX5JcE/zAs/zyhgJY7NqPecJjchCcE3ITkOdn2YxXsUYmqzQIDAQAB +AoGATaLcI7MSgOwqggPFVyu+nS2AiruHAOkSPZ/tg9daFznISRegaK6/bL+d1vjT +lWFi8ugxQV0EEK710XHpzldQAH0YQ9YA86s7P/a4SjETdRChFYt+CV+aZ4feyNPV +OZcKuoE82MUFU03jaJsWJJ4jybPPTcZ0Rr25oFpkR2fnPMECQQDXrRWviHuLkmrV +WqZQLXiPt6bCrTowpnKo62Un5yrA7dehfL9b12J7/9tgfy0ZHXZtBXSHlELZ4LeA +wpYfsq59AkEAyj99EE++kU2QbkCqYKJ5xBQxNc2ntZ/EfiBXMuQNwncj4m86xFLj +coFHyOrRjo1GZAFxsZsNbf74xgMLDxIOkQJAHzKiWGndtSrQ2Vvrgt2Q+vkN3ktA +h5kMLPMgBs2hmZbOAkYRSC+3x0gTa7n5xBBG+S441QPVR78BzFZZcOxf4QJBAIVk +lH8iqYU6jE07l2Q/JWK/Eqny529yXe32NK0bHzwoymE5jaAZL2zBefA5eFe2NDwX +e75xjs0Cw2AOd8fL2BECQBqAafyDcER1SasqV5hkjyFQQu8FqzLUyppwbTKTRZji +s+xxGwP9jT9LaKC6w9nuzFmaGi0OO3ciE+I+X89YiKc= -----END RSA PRIVATE KEY----- From 8a503526e455b981718c2d24bd1630fd5b4cf7ac Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 15 Sep 2016 09:22:52 -0400 Subject: [PATCH 175/871] pg_buffercache: Allow huge allocations. Otherwise, users who have configured shared_buffers >= 256GB won't be able to use this module. There probably aren't many of those, but it doesn't hurt anything to fix it so that it works. Backpatch to 9.4, where MemoryContextAllocHuge was introduced. The same problem exists in older branches, but there's no easy way to fix it there. KaiGai Kohei --- contrib/pg_buffercache/pg_buffercache_pages.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index 17b4b6fa6f..da13bde359 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -124,7 +124,9 @@ pg_buffercache_pages(PG_FUNCTION_ARGS) fctx->tupdesc = BlessTupleDesc(tupledesc); /* Allocate NBuffers worth of BufferCachePagesRec records. */ - fctx->record = (BufferCachePagesRec *) palloc(sizeof(BufferCachePagesRec) * NBuffers); + fctx->record = (BufferCachePagesRec *) + MemoryContextAllocHuge(CurrentMemoryContext, + sizeof(BufferCachePagesRec) * NBuffers); /* Set max calls and remember the user function context. */ funcctx->max_calls = NBuffers; From 5472ed3e9bc23eff0b4e457fe564ac667cb69441 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 15 Sep 2016 11:23:25 -0400 Subject: [PATCH 176/871] Make min_parallel_relation_size's default value platform-independent. The documentation states that the default value is 8MB, but this was only true at BLCKSZ = 8kB, because the default was hard-coded as 1024. Make the code match the docs by computing the default as 8MB/BLCKSZ. Oversight in commit 75be66464, noted pursuant to a gripe from Peter E. Discussion: <90634e20-097a-e4fd-67d5-fb2c42f0dd71@2ndquadrant.com> --- src/backend/utils/misc/guc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index c72bd6190a..ce4eef950b 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2758,7 +2758,7 @@ static struct config_int ConfigureNamesInt[] = GUC_UNIT_BLOCKS, }, &min_parallel_relation_size, - 1024, 0, INT_MAX / 3, + (8 * 1024 * 1024) / BLCKSZ, 0, INT_MAX / 3, NULL, NULL, NULL }, From ffccee473682ed18a27d667b7a4f45d802dd61c4 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 15 Sep 2016 11:35:57 -0400 Subject: [PATCH 177/871] Fix typo in comment. Amit Langote --- src/backend/executor/execIndexing.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/executor/execIndexing.c b/src/backend/executor/execIndexing.c index 0e2d834ed1..009c1b7481 100644 --- a/src/backend/executor/execIndexing.c +++ b/src/backend/executor/execIndexing.c @@ -2,7 +2,7 @@ * * execIndexing.c * routines for inserting index tuples and enforcing unique and - * exclusive constraints. + * exclusion constraints. * * ExecInsertIndexTuples() is the main entry point. It's called after * inserting a tuple to the heap, and it inserts corresponding index tuples From 5c6df67e0c961f68e73e7c1e6312211ed59da00a Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 15 Sep 2016 22:29:39 +0300 Subject: [PATCH 178/871] Fix building with LibreSSL. LibreSSL defines OPENSSL_VERSION_NUMBER to claim that it is version 2.0.0, but it doesn't have the functions added in OpenSSL 1.1.0. Add autoconf checks for the individual functions we need, and stop relying on OPENSSL_VERSION_NUMBER. Backport to 9.5 and 9.6, like the patch that broke this. In the back-branches, there are still a few OPENSSL_VERSION_NUMBER checks left, to check for OpenSSL 0.9.8 or 0.9.7. I left them as they were - LibreSSL has all those functions, so they work as intended. Per buildfarm member curculio. Discussion: <2442.1473957669@sss.pgh.pa.us> --- configure | 31 ++++++++++++++++++++++++ configure.in | 10 ++++++++ contrib/pgcrypto/openssl.c | 10 +++++--- src/backend/libpq/be-secure-openssl.c | 6 ++--- src/include/pg_config.h.in | 18 ++++++++++++++ src/interfaces/libpq/fe-secure-openssl.c | 31 +++++++++++++----------- 6 files changed, 85 insertions(+), 21 deletions(-) diff --git a/configure b/configure index caf6f260ee..5fc8c442a2 100755 --- a/configure +++ b/configure @@ -9711,6 +9711,37 @@ if test "x$ac_cv_func_SSL_get_current_compression" = xyes; then : #define HAVE_SSL_GET_CURRENT_COMPRESSION 1 _ACEOF +fi +done + + # Functions introduced in OpenSSL 1.1.0. We used to check for + # OPENSSL_VERSION_NUMBER, but that didn't work with 1.1.0, because LibreSSL + # defines OPENSSL_VERSION_NUMBER to claim version 2.0.0, even though it + # doesn't have these OpenSSL 1.1.0 functions. So check for individual + # functions. + for ac_func in OPENSSL_init_ssl BIO_get_data BIO_meth_new ASN1_STRING_get0_data RAND_OpenSSL +do : + as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` +ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" +if eval test \"x\$"$as_ac_var"\" = x"yes"; then : + cat >>confdefs.h <<_ACEOF +#define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 +_ACEOF + +fi +done + + # OpenSSL versions before 1.1.0 required setting callback functions, for + # thread-safety. In 1.1.0, it's no longer required, and CRYPTO_lock() + # function was removed. + for ac_func in CRYPTO_lock +do : + ac_fn_c_check_func "$LINENO" "CRYPTO_lock" "ac_cv_func_CRYPTO_lock" +if test "x$ac_cv_func_CRYPTO_lock" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_CRYPTO_LOCK 1 +_ACEOF + fi done diff --git a/configure.in b/configure.in index c42680607a..96d865de9f 100644 --- a/configure.in +++ b/configure.in @@ -1118,6 +1118,16 @@ if test "$with_openssl" = yes ; then AC_SEARCH_LIBS(SSL_new, ssleay32 ssl, [], [AC_MSG_ERROR([library 'ssleay32' or 'ssl' is required for OpenSSL])]) fi AC_CHECK_FUNCS([SSL_get_current_compression]) + # Functions introduced in OpenSSL 1.1.0. We used to check for + # OPENSSL_VERSION_NUMBER, but that didn't work with 1.1.0, because LibreSSL + # defines OPENSSL_VERSION_NUMBER to claim version 2.0.0, even though it + # doesn't have these OpenSSL 1.1.0 functions. So check for individual + # functions. + AC_CHECK_FUNCS([OPENSSL_init_ssl BIO_get_data BIO_meth_new ASN1_STRING_get0_data RAND_OpenSSL]) + # OpenSSL versions before 1.1.0 required setting callback functions, for + # thread-safety. In 1.1.0, it's no longer required, and CRYPTO_lock() + # function was removed. + AC_CHECK_FUNCS([CRYPTO_lock]) fi if test "$with_pam" = yes ; then diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index e264aa49df..851cb616cb 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -914,10 +914,6 @@ px_find_cipher(const char *name, PX_Cipher **res) static int openssl_random_init = 0; -#if OPENSSL_VERSION_NUMBER < 0x10100000L -#define RAND_OpenSSL RAND_SSLeay -#endif - /* * OpenSSL random should re-feeded occasionally. From /dev/urandom * preferably. @@ -926,7 +922,13 @@ static void init_openssl_rand(void) { if (RAND_get_rand_method() == NULL) + { +#ifdef HAVE_RAND_OPENSSL RAND_set_rand_method(RAND_OpenSSL()); +#else + RAND_set_rand_method(RAND_SSLeay()); +#endif + } openssl_random_init = 1; } diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index 2afd738945..fedb02cd82 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -165,7 +165,7 @@ be_tls_init(void) if (!SSL_context) { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L +#ifdef HAVE_OPENSSL_INIT_SSL OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL); #else OPENSSL_config(NULL); @@ -672,7 +672,7 @@ be_tls_write(Port *port, void *ptr, size_t len, int *waitfor) * to retry; do we need to adopt their logic for that? */ -#if OPENSSL_VERSION_NUMBER < 0x10100000L +#ifndef HAVE_BIO_GET_DATA #define BIO_get_data(bio) (bio->ptr) #define BIO_set_data(bio, data) (bio->ptr = data) #endif @@ -726,7 +726,7 @@ my_BIO_s_socket(void) if (!my_bio_methods) { BIO_METHOD *biom = (BIO_METHOD *) BIO_s_socket(); -#if OPENSSL_VERSION_NUMBER >= 0x10100000L +#ifdef HAVE_BIO_METH_NEW int my_bio_index; my_bio_index = BIO_get_new_index(); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index b621ff2af5..7dbfa90bf4 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -84,12 +84,21 @@ /* Define to 1 if you have the `append_history' function. */ #undef HAVE_APPEND_HISTORY +/* Define to 1 if you have the `ASN1_STRING_get0_data' function. */ +#undef HAVE_ASN1_STRING_GET0_DATA + /* Define to 1 if you want to use atomics if available. */ #undef HAVE_ATOMICS /* Define to 1 if you have the header file. */ #undef HAVE_ATOMIC_H +/* Define to 1 if you have the `BIO_get_data' function. */ +#undef HAVE_BIO_GET_DATA + +/* Define to 1 if you have the `BIO_meth_new' function. */ +#undef HAVE_BIO_METH_NEW + /* Define to 1 if you have the `cbrt' function. */ #undef HAVE_CBRT @@ -102,6 +111,9 @@ /* Define to 1 if you have the `crypt' function. */ #undef HAVE_CRYPT +/* Define to 1 if you have the `CRYPTO_lock' function. */ +#undef HAVE_CRYPTO_LOCK + /* Define to 1 if you have the header file. */ #undef HAVE_CRYPT_H @@ -364,6 +376,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_NET_IF_H +/* Define to 1 if you have the `OPENSSL_init_ssl' function. */ +#undef HAVE_OPENSSL_INIT_SSL + /* Define to 1 if you have the header file. */ #undef HAVE_OSSP_UUID_H @@ -403,6 +418,9 @@ /* Define to 1 if you have the `random' function. */ #undef HAVE_RANDOM +/* Define to 1 if you have the `RAND_OpenSSL' function. */ +#undef HAVE_RAND_OPENSSL + /* Define to 1 if you have the header file. */ #undef HAVE_READLINE_H diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index fe81825144..2dcdce7265 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -506,10 +506,6 @@ wildcard_certificate_match(const char *pattern, const char *string) return 1; } -#if OPENSSL_VERSION_NUMBER < 0x10100000L -#define ASN1_STRING_get0_data ASN1_STRING_data -#endif - /* * Check if a name from a server's certificate matches the peer's hostname. * @@ -544,7 +540,11 @@ verify_peer_name_matches_certificate_name(PGconn *conn, ASN1_STRING *name_entry, * There is no guarantee the string returned from the certificate is * NULL-terminated, so make a copy that is. */ +#ifdef HAVE_ASN1_STRING_GET0_DATA namedata = ASN1_STRING_get0_data(name_entry); +#else + namedata = ASN1_STRING_data(name_entry); +#endif len = ASN1_STRING_length(name_entry); name = malloc(len + 1); if (name == NULL) @@ -732,10 +732,13 @@ verify_peer_name_matches_certificate(PGconn *conn) return found_match && !got_error; } -#if defined(ENABLE_THREAD_SAFETY) && OPENSSL_VERSION_NUMBER < 0x10100000L +#if defined(ENABLE_THREAD_SAFETY) && defined(HAVE_CRYPTO_LOCK) /* - * Callback functions for OpenSSL internal locking. (OpenSSL 1.1.0 - * does its own locking, and doesn't need these anymore.) + * Callback functions for OpenSSL internal locking. (OpenSSL 1.1.0 + * does its own locking, and doesn't need these anymore. The + * CRYPTO_lock() function was removed in 1.1.0, when the callbacks + * were made obsolete, so we assume that if CRYPTO_lock() exists, + * the callbacks are still required.) */ static unsigned long @@ -765,7 +768,7 @@ pq_lockingcallback(int mode, int n, const char *file, int line) PGTHREAD_ERROR("failed to unlock mutex"); } } -#endif /* ENABLE_THREAD_SAFETY && OPENSSL_VERSION_NUMBER < 0x10100000L */ +#endif /* ENABLE_THREAD_SAFETY && HAVE_CRYPTO_LOCK */ /* * Initialize SSL system, in particular creating the SSL_context object @@ -804,7 +807,7 @@ pgtls_init(PGconn *conn) if (pthread_mutex_lock(&ssl_config_mutex)) return -1; -#if OPENSSL_VERSION_NUMBER < 0x10100000L +#ifdef HAVE_CRYPTO_LOCK if (pq_init_crypto_lib) { /* @@ -845,14 +848,14 @@ pgtls_init(PGconn *conn) CRYPTO_set_locking_callback(pq_lockingcallback); } } -#endif /* OPENSSL_VERSION_NUMBER < 0x10100000L */ +#endif /* HAVE_CRYPTO_LOCK */ #endif /* ENABLE_THREAD_SAFETY */ if (!SSL_context) { if (pq_init_ssl_lib) { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L +#ifdef HAVE_OPENSSL_INIT_SSL OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL); #else OPENSSL_config(NULL); @@ -913,7 +916,7 @@ pgtls_init(PGconn *conn) static void destroy_ssl_system(void) { -#if defined(ENABLE_THREAD_SAFETY) && OPENSSL_VERSION_NUMBER < 0x10100000L +#if defined(ENABLE_THREAD_SAFETY) && defined(HAVE_CRYPTO_LOCK) /* Mutex is created in initialize_ssl_system() */ if (pthread_mutex_lock(&ssl_config_mutex)) return; @@ -1628,7 +1631,7 @@ PQsslAttribute(PGconn *conn, const char *attribute_name) * to retry; do we need to adopt their logic for that? */ -#if OPENSSL_VERSION_NUMBER < 0x10100000L +#ifndef HAVE_BIO_GET_DATA #define BIO_get_data(bio) (bio->ptr) #define BIO_set_data(bio, data) (bio->ptr = data) #endif @@ -1701,7 +1704,7 @@ my_BIO_s_socket(void) if (!my_bio_methods) { BIO_METHOD *biom = (BIO_METHOD *) BIO_s_socket(); -#if OPENSSL_VERSION_NUMBER >= 0x10100000L +#ifdef HAVE_BIO_METH_NEW int my_bio_index; my_bio_index = BIO_get_new_index(); From 5225c66336a1e4b46925e9f169086fc70f49736f Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 15 Sep 2016 17:24:54 -0400 Subject: [PATCH 179/871] Clarify policy on marking inherited constraints as valid. Amit Langote and Robert Haas --- doc/src/sgml/ref/alter_table.sgml | 14 +++++++++----- src/backend/commands/tablecmds.c | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 6f51cbc896..e48ccf21e4 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -1028,11 +1028,15 @@ ALTER TABLE ALL IN TABLESPACE name If a table has any descendant tables, it is not permitted to add, - rename, or change the type of a column, or rename an inherited constraint - in the parent table without doing - the same to the descendants. That is, ALTER TABLE ONLY - will be rejected. This ensures that the descendants always have - columns matching the parent. + rename, or change the type of a column in the parent table without doing + same to the descendants. This ensures that the descendants always have + columns matching the parent. Similarly, a constraint cannot be renamed + in the parent without also renaming it in all descendents, so that + constraints also match between the parent and its descendents. + Also, because selecting from the parent also selects from its descendents, + a constraint on the parent cannot be marked valid unless it is also marked + valid for those descendents. In all of these cases, ALTER TABLE + ONLY will be rejected. diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 86e98148c1..d31276284c 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -6908,7 +6908,8 @@ ATExecValidateConstraint(Relation rel, char *constrName, bool recurse, /* * If we are told not to recurse, there had better not be any - * child tables; else the addition would put them out of step. + * child tables, because we can't mark the constraint on the + * parent valid unless it is valid for all child tables. */ if (!recurse) ereport(ERROR, From d8c61c9765339351409f06bbf964dcb8c1929e8b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 16 Sep 2016 09:36:19 -0400 Subject: [PATCH 180/871] Add debugging aid "bmsToString(Bitmapset *bms)". This function has no direct callers at present, but it's convenient for manual use in a debugger, rather than having to inspect memory and do bit-counting in your head. In passing, get rid of useless outBitmapset() wrapper around _outBitmapset(); let's just export the function that does the work. Likewise for outToken(). Ashutosh Bapat, tweaked a bit by me Discussion: --- src/backend/nodes/outfuncs.c | 61 ++++++++++++++++++------------------ src/include/nodes/nodes.h | 5 +-- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 7e092d700c..ae869547f3 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -81,7 +81,7 @@ /* Write a character-string (possibly NULL) field */ #define WRITE_STRING_FIELD(fldname) \ (appendStringInfo(str, " :" CppAsString(fldname) " "), \ - _outToken(str, node->fldname)) + outToken(str, node->fldname)) /* Write a parse location field (actually same as INT case) */ #define WRITE_LOCATION_FIELD(fldname) \ @@ -95,21 +95,21 @@ /* Write a bitmapset field */ #define WRITE_BITMAPSET_FIELD(fldname) \ (appendStringInfo(str, " :" CppAsString(fldname) " "), \ - _outBitmapset(str, node->fldname)) + outBitmapset(str, node->fldname)) #define booltostr(x) ((x) ? "true" : "false") /* - * _outToken + * outToken * Convert an ordinary string (eg, an identifier) into a form that * will be decoded back to a plain token by read.c's functions. * * If a null or empty string is given, it is encoded as "<>". */ -static void -_outToken(StringInfo str, const char *s) +void +outToken(StringInfo str, const char *s) { if (s == NULL || *s == '\0') { @@ -140,13 +140,6 @@ _outToken(StringInfo str, const char *s) } } -/* for use by extensions which define extensible nodes */ -void -outToken(StringInfo str, const char *s) -{ - _outToken(str, s); -} - static void _outList(StringInfo str, const List *node) { @@ -185,13 +178,13 @@ _outList(StringInfo str, const List *node) } /* - * _outBitmapset - + * outBitmapset - * converts a bitmap set of integers * * Note: the output format is "(b int int ...)", similar to an integer List. */ -static void -_outBitmapset(StringInfo str, const Bitmapset *bms) +void +outBitmapset(StringInfo str, const Bitmapset *bms) { int x; @@ -203,13 +196,6 @@ _outBitmapset(StringInfo str, const Bitmapset *bms) appendStringInfoChar(str, ')'); } -/* for use by extensions which define extensible nodes */ -void -outBitmapset(StringInfo str, const Bitmapset *bms) -{ - _outBitmapset(str, bms); -} - /* * Print the value of a Datum given its type. */ @@ -632,7 +618,7 @@ _outCustomScan(StringInfo str, const CustomScan *node) WRITE_BITMAPSET_FIELD(custom_relids); /* CustomName is a key to lookup CustomScanMethods */ appendStringInfoString(str, " :methods "); - _outToken(str, node->methods->CustomName); + outToken(str, node->methods->CustomName); } static void @@ -1196,7 +1182,7 @@ _outBoolExpr(StringInfo str, const BoolExpr *node) break; } appendStringInfoString(str, " :boolop "); - _outToken(str, opstr); + outToken(str, opstr); WRITE_NODE_FIELD(args); WRITE_LOCATION_FIELD(location); @@ -1609,14 +1595,14 @@ _outPathInfo(StringInfo str, const Path *node) { WRITE_ENUM_FIELD(pathtype, NodeTag); appendStringInfoString(str, " :parent_relids "); - _outBitmapset(str, node->parent->relids); + outBitmapset(str, node->parent->relids); if (node->pathtarget != node->parent->reltarget) WRITE_NODE_FIELD(pathtarget); appendStringInfoString(str, " :required_outer "); if (node->param_info) - _outBitmapset(str, node->param_info->ppi_req_outer); + outBitmapset(str, node->param_info->ppi_req_outer); else - _outBitmapset(str, NULL); + outBitmapset(str, NULL); WRITE_BOOL_FIELD(parallel_aware); WRITE_BOOL_FIELD(parallel_safe); WRITE_INT_FIELD(parallel_workers); @@ -1740,7 +1726,7 @@ _outCustomPath(StringInfo str, const CustomPath *node) WRITE_NODE_FIELD(custom_paths); WRITE_NODE_FIELD(custom_private); appendStringInfoString(str, " :methods "); - _outToken(str, node->methods->CustomName); + outToken(str, node->methods->CustomName); } static void @@ -2994,12 +2980,12 @@ _outValue(StringInfo str, const Value *value) case T_String: /* - * We use _outToken to provide escaping of the string's content, + * We use outToken to provide escaping of the string's content, * but we don't want it to do anything with an empty string. */ appendStringInfoChar(str, '"'); if (value->val.str[0] != '\0') - _outToken(str, value->val.str); + outToken(str, value->val.str); appendStringInfoChar(str, '"'); break; case T_BitString: @@ -3895,3 +3881,18 @@ nodeToString(const void *obj) outNode(&str, obj); return str.data; } + +/* + * bmsToString - + * returns the ascii representation of the Bitmapset as a palloc'd string + */ +char * +bmsToString(const Bitmapset *bms) +{ + StringInfoData str; + + /* see stringinfo.h for an explanation of this maneuver */ + initStringInfo(&str); + outBitmapset(&str, bms); + return str.data; +} diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 2f7efa810c..88297bbe80 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -551,16 +551,17 @@ extern PGDLLIMPORT Node *newNodeMacroHolder; /* * nodes/{outfuncs.c,print.c} */ -extern char *nodeToString(const void *obj); - struct Bitmapset; /* not to include bitmapset.h here */ struct StringInfoData; /* not to include stringinfo.h here */ + extern void outNode(struct StringInfoData *str, const void *obj); extern void outToken(struct StringInfoData *str, const char *s); extern void outBitmapset(struct StringInfoData *str, const struct Bitmapset *bms); extern void outDatum(struct StringInfoData *str, uintptr_t value, int typlen, bool typbyval); +extern char *nodeToString(const void *obj); +extern char *bmsToString(const struct Bitmapset *bms); /* * nodes/{readfuncs.c,read.c} From 3fcc98c990ede7d3b415cc06eb6664aac6e7bbbc Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sun, 18 Sep 2016 13:46:32 +0300 Subject: [PATCH 181/871] Fix ecpg -? option on Windows, add -V alias for --version. This makes the -? and -V options work consistently with other binaries. --help and --version are now only recognized as the first option, i.e. "ecpg --foobar --help" no longer prints the help, but that's consistent with most of our other binaries, too. Backpatch to all supported versions. Haribabu Kommi Discussion: --- src/interfaces/ecpg/preproc/ecpg.c | 45 +++++++++++------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/src/interfaces/ecpg/preproc/ecpg.c b/src/interfaces/ecpg/preproc/ecpg.c index 3ce9d04bcc..46bacb0e8e 100644 --- a/src/interfaces/ecpg/preproc/ecpg.c +++ b/src/interfaces/ecpg/preproc/ecpg.c @@ -54,7 +54,7 @@ help(const char *progname) " \"no_indicator\", \"prepare\", \"questionmarks\"\n")); printf(_(" --regression run in regression testing mode\n")); printf(_(" -t turn on autocommit of transactions\n")); - printf(_(" --version output version information, then exit\n")); + printf(_(" -V, --version output version information, then exit\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nIf no output file is specified, the name is formed by adding .c to the\n" "input file name, after stripping off .pgc if present.\n")); @@ -111,15 +111,11 @@ add_preprocessor_define(char *define) defines->next = pd; } -#define ECPG_GETOPT_LONG_HELP 1 -#define ECPG_GETOPT_LONG_VERSION 2 -#define ECPG_GETOPT_LONG_REGRESSION 3 +#define ECPG_GETOPT_LONG_REGRESSION 1 int main(int argc, char *const argv[]) { static struct option ecpg_options[] = { - {"help", no_argument, NULL, ECPG_GETOPT_LONG_HELP}, - {"version", no_argument, NULL, ECPG_GETOPT_LONG_VERSION}, {"regression", no_argument, NULL, ECPG_GETOPT_LONG_REGRESSION}, {NULL, 0, NULL, 0} }; @@ -144,32 +140,25 @@ main(int argc, char *const argv[]) return (ILLEGAL_OPTION); } + if (argc > 1) + { + if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) + { + help(progname); + exit(0); + } + if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) + { + printf("ecpg %s\n", PG_VERSION); + exit(0); + } + } + output_filename = NULL; - while ((c = getopt_long(argc, argv, "vcio:I:tD:dC:r:h?", ecpg_options, NULL)) != -1) + while ((c = getopt_long(argc, argv, "vcio:I:tD:dC:r:h", ecpg_options, NULL)) != -1) { switch (c) { - case ECPG_GETOPT_LONG_VERSION: - printf("ecpg %s\n", PG_VERSION); - exit(0); - case ECPG_GETOPT_LONG_HELP: - help(progname); - exit(0); - - /* - * -? is an alternative spelling of --help. However it is also - * returned by getopt_long for unknown options. We can - * distinguish both cases by means of the optopt variable - * which is set to 0 if it was really -? and not an unknown - * option character. - */ - case '?': - if (optopt == 0) - { - help(progname); - exit(0); - } - break; case ECPG_GETOPT_LONG_REGRESSION: regression_mode = true; break; From 2c8f0d6e53e5dbcf28ee127303b81a6e12942665 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 19 Sep 2016 13:38:21 -0400 Subject: [PATCH 182/871] Update recovery_min_apply_delay docs for remote_apply mode. Bernd Helmle, reviewed by Thomas Munro, tweaked by me. --- doc/src/sgml/recovery-config.sgml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml index de3fb10f5b..8c24ae2174 100644 --- a/doc/src/sgml/recovery-config.sgml +++ b/doc/src/sgml/recovery-config.sgml @@ -487,10 +487,17 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows This parameter is intended for use with streaming replication deployments; however, if the parameter is specified it will be honored in all cases. - Synchronous replication is not affected by this setting because there is - not yet any setting to request synchronous apply of transaction commits. + hot_standby_feedback will be delayed by use of this feature which could lead to bloat on the master; use both together with care. + + + + Synchronous replication is affected by this setting when synchronous_commit + is set to remote_apply; every COMMIT + will need to wait to be applied. + + From 8614b39bca8fc64b1128c7911424647c9a1d1da1 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 19 Sep 2016 14:21:48 -0400 Subject: [PATCH 183/871] MSVC: Include pg_recvlogical in client-only install. MauMau, reviewed by Michael Paquier --- src/tools/msvc/Install.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/msvc/Install.pm b/src/tools/msvc/Install.pm index 52dde65a0e..e04efe6559 100644 --- a/src/tools/msvc/Install.pm +++ b/src/tools/msvc/Install.pm @@ -24,8 +24,8 @@ my @client_program_files = ( 'dropdb', 'droplang', 'dropuser', 'ecpg', 'libecpg', 'libecpg_compat', 'libpgtypes', 'libpq', 'pg_basebackup', 'pg_config', 'pg_dump', 'pg_dumpall', - 'pg_isready', 'pg_receivexlog', 'pg_restore', 'psql', - 'reindexdb', 'vacuumdb', @client_contribs); + 'pg_isready', 'pg_receivexlog', 'pg_recvlogical', 'pg_restore', + 'psql', 'reindexdb', 'vacuumdb', @client_contribs); sub lcopy { From 6cc54f38a9fe1f4103c45a9858804d1d5d4de0fd Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Sep 2016 21:56:16 +0300 Subject: [PATCH 184/871] Remove obsolete warning from docs. Python 2.4 and Fedora 4 are both obsolete at this point, especially unpatched debug builds. Discussion: <85e377b2-d459-396e-59b1-115548bbc059@iki.fi> --- doc/src/sgml/plpython.sgml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/doc/src/sgml/plpython.sgml b/doc/src/sgml/plpython.sgml index 905e757ab6..bb69c752b8 100644 --- a/doc/src/sgml/plpython.sgml +++ b/doc/src/sgml/plpython.sgml @@ -696,19 +696,6 @@ AS $$ $$ LANGUAGE plpythonu; - - - Due to Python - bug #1483133, - some debug versions of Python 2.4 - (configured and compiled with option --with-pydebug) - are known to crash the PostgreSQL server - when using an iterator to return a set result. - Unpatched versions of Fedora 4 contain this bug. - It does not happen in production versions of Python or on patched - versions of Fedora 4. - - From 40c3fe4980e73acb0db75a3c737a4a52e09d4cf4 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Sep 2016 22:55:43 +0300 Subject: [PATCH 185/871] Fix latency calculation when there are \sleep commands in the script. We can't use txn_scheduled to hold the sleep-until time for \sleep, because that interferes with calculation of the latency of the transaction as whole. Backpatch to 9.4, where this bug was introduced. Fabien COELHO Discussion: --- src/bin/pgbench/pgbench.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 56c37d537e..4676a59020 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -250,6 +250,7 @@ typedef struct int nvariables; /* number of variables */ bool vars_sorted; /* are variables sorted by name? */ int64 txn_scheduled; /* scheduled start time of transaction (usec) */ + int64 sleep_until; /* scheduled start time of next cmd (usec) */ instr_time txn_begin; /* used for measuring schedule lag times */ instr_time stmt_begin; /* used for measuring statement latencies */ int use_file; /* index in sql_scripts for this client */ @@ -1830,6 +1831,7 @@ doCustom(TState *thread, CState *st, StatsData *agg) } } + st->sleep_until = st->txn_scheduled; st->sleeping = true; st->throttling = true; st->is_throttled = true; @@ -1842,7 +1844,7 @@ doCustom(TState *thread, CState *st, StatsData *agg) { /* are we sleeping? */ if (INSTR_TIME_IS_ZERO(now)) INSTR_TIME_SET_CURRENT(now); - if (INSTR_TIME_GET_MICROSEC(now) < st->txn_scheduled) + if (INSTR_TIME_GET_MICROSEC(now) < st->sleep_until) return true; /* Still sleeping, nothing to do here */ /* Else done sleeping, go ahead with next command */ st->sleeping = false; @@ -2141,7 +2143,7 @@ doCustom(TState *thread, CState *st, StatsData *agg) usec *= 1000000; INSTR_TIME_SET_CURRENT(now); - st->txn_scheduled = INSTR_TIME_GET_MICROSEC(now) + usec; + st->sleep_until = INSTR_TIME_GET_MICROSEC(now) + usec; st->sleeping = true; st->listen = true; From 45310221a9afccd98e78813459472370ade9dc4c Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 20 Sep 2016 11:38:25 +0300 Subject: [PATCH 186/871] Fix outdated comments, GIST search queue is not an RBTree anymore. The GiST search queue is implemented as a pairing heap rather than as Red-Black Tree, since 9.5 (commit e7032610). I neglected these comments in that commit. --- src/backend/access/gist/gistscan.c | 4 ++-- src/include/access/gist_private.h | 18 +++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/backend/access/gist/gistscan.c b/src/backend/access/gist/gistscan.c index ba611ee490..2526a3965c 100644 --- a/src/backend/access/gist/gistscan.c +++ b/src/backend/access/gist/gistscan.c @@ -125,7 +125,7 @@ gistrescan(IndexScanDesc scan, ScanKey key, int nkeys, * which is created on the second call and reset on later calls. Thus, in * the common case where a scan is only rescan'd once, we just put the * queue in scanCxt and don't pay the overhead of making a second memory - * context. If we do rescan more than once, the first RBTree is just left + * context. If we do rescan more than once, the first queue is just left * for dead until end of scan; this small wastage seems worth the savings * in the common case. */ @@ -181,7 +181,7 @@ gistrescan(IndexScanDesc scan, ScanKey key, int nkeys, ALLOCSET_DEFAULT_SIZES); } - /* create new, empty RBTree for search queue */ + /* create new, empty pairing heap for search queue */ oldCxt = MemoryContextSwitchTo(so->queueCxt); so->queue = pairingheap_allocate(pairingheap_GISTSearchItem_cmp, scan); MemoryContextSwitchTo(oldCxt); diff --git a/src/include/access/gist_private.h b/src/include/access/gist_private.h index 1231585017..78e87a6077 100644 --- a/src/include/access/gist_private.h +++ b/src/include/access/gist_private.h @@ -107,15 +107,11 @@ typedef struct GISTSTATE * upper index pages; this rule avoids doing extra work during a search that * ends early due to LIMIT. * - * To perform an ordered search, we use an RBTree to manage the distance-order - * queue. Each GISTSearchTreeItem stores all unvisited items of the same - * distance; they are GISTSearchItems chained together via their next fields. - * - * In a non-ordered search (no order-by operators), the RBTree degenerates - * to a single item, which we use as a queue of unvisited index pages only. - * In this case matched heap items from the current index leaf page are - * remembered in GISTScanOpaqueData.pageData[] and returned directly from - * there, instead of building a separate GISTSearchItem for each one. + * To perform an ordered search, we use a pairing heap to manage the + * distance-order queue. In a non-ordered search (no order-by operators), + * we use it to return heap tuples before unvisited index pages, to + * ensure depth-first order, but all entries are otherwise considered + * equal. */ /* Individual heap tuple to be visited */ @@ -298,8 +294,8 @@ typedef struct #define GIST_ROOT_BLKNO 0 /* - * Before PostgreSQL 9.1, we used rely on so-called "invalid tuples" on inner - * pages to finish crash recovery of incomplete page splits. If a crash + * Before PostgreSQL 9.1, we used to rely on so-called "invalid tuples" on + * inner pages to finish crash recovery of incomplete page splits. If a crash * happened in the middle of a page split, so that the downlink pointers were * not yet inserted, crash recovery inserted a special downlink pointer. The * semantics of an invalid tuple was that it if you encounter one in a scan, From 419113dfdc4c729f6c763cc30a9b02ee68a7da94 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 20 Sep 2016 12:04:41 -0400 Subject: [PATCH 187/871] Retry DSM control segment creation if Windows indicates access denied. Otherwise, attempts to run multiple postmasters running on the same machine may fail, because Windows sometimes returns ERROR_ACCESS_DENIED rather than ERROR_ALREADY_EXISTS when there is an existing segment. Hitting this bug is much more likely because of another defect not fixed by this patch, namely that dsm_postmaster_startup() uses random() which returns the same value every time. But that's not a reason not to fix this. Kyotaro Horiguchi and Amit Kapila, reviewed by Michael Paquier Discussion: --- src/backend/storage/ipc/dsm_impl.c | 31 +++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/backend/storage/ipc/dsm_impl.c b/src/backend/storage/ipc/dsm_impl.c index 99051f02c8..7122c4abe0 100644 --- a/src/backend/storage/ipc/dsm_impl.c +++ b/src/backend/storage/ipc/dsm_impl.c @@ -671,6 +671,7 @@ dsm_impl_windows(dsm_op op, dsm_handle handle, Size request_size, { DWORD size_high; DWORD size_low; + DWORD errcode; /* Shifts >= the width of the type are undefined. */ #ifdef _WIN64 @@ -686,27 +687,31 @@ dsm_impl_windows(dsm_op op, dsm_handle handle, Size request_size, size_high, /* Upper 32 bits of size */ size_low, /* Lower 32 bits of size */ name); + + errcode = GetLastError(); + if (errcode == ERROR_ALREADY_EXISTS || errcode == ERROR_ACCESS_DENIED) + { + /* + * On Windows, when the segment already exists, a handle for the + * existing segment is returned. We must close it before + * returning. However, if the existing segment is created by a + * service, then it returns ERROR_ACCESS_DENIED. We don't do + * _dosmaperr here, so errno won't be modified. + */ + if (hmap) + CloseHandle(hmap); + return false; + } + if (!hmap) { - _dosmaperr(GetLastError()); + _dosmaperr(errcode); ereport(elevel, (errcode_for_dynamic_shared_memory(), errmsg("could not create shared memory segment \"%s\": %m", name))); return false; } - _dosmaperr(GetLastError()); - if (errno == EEXIST) - { - /* - * On Windows, when the segment already exists, a handle for the - * existing segment is returned. We must close it before - * returning. We don't do _dosmaperr here, so errno won't be - * modified. - */ - CloseHandle(hmap); - return false; - } } else { From 470d886c32efafa1b068b5ca48afafc2198c68d4 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 20 Sep 2016 12:24:44 -0400 Subject: [PATCH 188/871] Use PostmasterRandom(), not random(), for DSM control segment ID. Otherwise, every startup gets the same "random" value, which is definitely not what was intended. --- src/backend/postmaster/postmaster.c | 3 +-- src/backend/storage/ipc/dsm.c | 3 ++- src/include/postmaster/postmaster.h | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index eaf3f61ecf..67ccdce764 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -403,7 +403,6 @@ static void processCancelRequest(Port *port, void *pkt); static int initMasks(fd_set *rmask); static void report_fork_failure_to_client(Port *port, int errnum); static CAC_state canAcceptConnections(void); -static long PostmasterRandom(void); static void RandomSalt(char *salt, int len); static void signal_child(pid_t pid, int signal); static bool SignalSomeChildren(int signal, int targets); @@ -5101,7 +5100,7 @@ RandomSalt(char *salt, int len) /* * PostmasterRandom */ -static long +long PostmasterRandom(void) { /* diff --git a/src/backend/storage/ipc/dsm.c b/src/backend/storage/ipc/dsm.c index d8066647a0..edafe4a3b9 100644 --- a/src/backend/storage/ipc/dsm.c +++ b/src/backend/storage/ipc/dsm.c @@ -36,6 +36,7 @@ #include "lib/ilist.h" #include "miscadmin.h" +#include "postmaster/postmaster.h" #include "storage/dsm.h" #include "storage/ipc.h" #include "storage/lwlock.h" @@ -181,7 +182,7 @@ dsm_postmaster_startup(PGShmemHeader *shim) { Assert(dsm_control_address == NULL); Assert(dsm_control_mapped_size == 0); - dsm_control_handle = random(); + dsm_control_handle = (dsm_handle) PostmasterRandom(); if (dsm_control_handle == 0) continue; if (dsm_impl_op(DSM_OP_CREATE, dsm_control_handle, segsize, diff --git a/src/include/postmaster/postmaster.h b/src/include/postmaster/postmaster.h index b2d7776f2a..ef06d5d04c 100644 --- a/src/include/postmaster/postmaster.h +++ b/src/include/postmaster/postmaster.h @@ -48,6 +48,7 @@ extern const char *progname; extern void PostmasterMain(int argc, char *argv[]) pg_attribute_noreturn(); extern void ClosePostmasterPorts(bool am_syslogger); +extern long PostmasterRandom(void); extern int MaxLivePostmasterChildren(void); From 90c9648212f7c3f0838fdaafe006ed40556a1364 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 20 Sep 2016 12:00:00 -0400 Subject: [PATCH 189/871] Re-add translation markers that were lost When win32security.c was moved from src/backend/port/win32/security.c, the message writing function was changed from write_stderr to log_error, but nls.mk was not updated. We could add log_error to GETTEXT_TRIGGERS, but it's also used in src/common/exec.c in a different way and that would create some confusion or a larger patch. For now, just put an explicit translation marker onto the strings that were previously translated. --- src/port/win32security.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/port/win32security.c b/src/port/win32security.c index ab9cd67748..2c9ca15a6d 100644 --- a/src/port/win32security.c +++ b/src/port/win32security.c @@ -65,7 +65,7 @@ pgwin32_is_admin(void) if (!OpenProcessToken(GetCurrentProcess(), TOKEN_READ, &AccessToken)) { - log_error("could not open process token: error code %lu\n", + log_error(_("could not open process token: error code %lu\n"), GetLastError()); exit(1); } @@ -86,7 +86,7 @@ pgwin32_is_admin(void) DOMAIN_ALIAS_RID_ADMINS, 0, 0, 0, 0, 0, 0, &AdministratorsSid)) { - log_error("could not get SID for Administrators group: error code %lu\n", + log_error(_("could not get SID for Administrators group: error code %lu\n"), GetLastError()); exit(1); } @@ -96,7 +96,7 @@ pgwin32_is_admin(void) DOMAIN_ALIAS_RID_POWER_USERS, 0, 0, 0, 0, 0, 0, &PowerUsersSid)) { - log_error("could not get SID for PowerUsers group: error code %lu\n", + log_error(_("could not get SID for PowerUsers group: error code %lu\n"), GetLastError()); exit(1); } From 4f6494cfd26c1dfe708c4598c11eea5fce168fd1 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 20 Sep 2016 12:00:00 -0400 Subject: [PATCH 190/871] doc: Correct ALTER USER MAPPING example The existing example threw an error. From: gabrielle --- doc/src/sgml/ref/alter_user_mapping.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/ref/alter_user_mapping.sgml b/doc/src/sgml/ref/alter_user_mapping.sgml index 3a908130d8..3be54afee5 100644 --- a/doc/src/sgml/ref/alter_user_mapping.sgml +++ b/doc/src/sgml/ref/alter_user_mapping.sgml @@ -89,9 +89,9 @@ ALTER USER MAPPING FOR { user_name Examples - Change the password for user mapping bob, server foo: + Change the password for user mapping bob, server foo: -ALTER USER MAPPING FOR bob SERVER foo OPTIONS (user 'bob', password 'public'); +ALTER USER MAPPING FOR bob SERVER foo OPTIONS (SET password 'public'); From 16d1adb35cf887325b7c5dbf473632d557065171 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 20 Sep 2016 12:00:00 -0400 Subject: [PATCH 191/871] doc: Fix documentation to match actual make output based on patch from Takeshi Ideriha --- doc/src/sgml/installation.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 14a6d57aea..f6de18ed2d 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -1510,7 +1510,7 @@ su - postgres will take a few minutes depending on your hardware. The last line displayed should be: -All of PostgreSQL is successfully made. Ready to install. +All of PostgreSQL successfully made. Ready to install. @@ -1523,7 +1523,7 @@ All of PostgreSQL is successfully made. Ready to install. The last line displayed should be: -PostgreSQL, contrib and HTML documentation successfully made. Ready to install. +PostgreSQL, contrib, and documentation successfully made. Ready to install. From 46b55e7f853dc0ef60ae3b1042b883fa4ffac95f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 20 Sep 2016 12:00:00 -0400 Subject: [PATCH 192/871] pg_restore: Add -N option to exclude schemas This is similar to the -N option in pg_dump, except that it doesn't take a pattern, just like the existing -n option in pg_restore. From: Michael Banck --- doc/src/sgml/ref/pg_restore.sgml | 18 +++++++++++++++++- src/bin/pg_dump/pg_backup.h | 1 + src/bin/pg_dump/pg_backup_archiver.c | 5 +++++ src/bin/pg_dump/pg_restore.c | 8 +++++++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index c9069193af..bd5b405314 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -302,7 +302,7 @@ - + @@ -314,6 +314,22 @@ + + + + + + Do not restore objects that are in the named schema. Multiple schemas + to be excluded may be specified with multiple + + + When both + + + diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 4afa92f5f6..0a28124cf6 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -99,6 +99,7 @@ typedef struct _restoreOptions SimpleStringList indexNames; SimpleStringList functionNames; SimpleStringList schemaNames; + SimpleStringList schemaExcludeNames; SimpleStringList triggerNames; SimpleStringList tableNames; diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 05bdbdbf02..a69b06f6d7 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -2751,6 +2751,11 @@ _tocEntryRequired(TocEntry *te, teSection curSection, RestoreOptions *ropt) return 0; } + if (ropt->schemaExcludeNames.head != NULL + && te->namespace + && simple_string_list_member(&ropt->schemaExcludeNames, te->namespace)) + return 0; + if (ropt->selTypes) { if (strcmp(te->desc, "TABLE") == 0 || diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index fb08e6bb8e..b21fd263b0 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -85,6 +85,7 @@ main(int argc, char **argv) {"data-only", 0, NULL, 'a'}, {"dbname", 1, NULL, 'd'}, {"exit-on-error", 0, NULL, 'e'}, + {"exclude-schema", 1, NULL, 'N'}, {"file", 1, NULL, 'f'}, {"format", 1, NULL, 'F'}, {"function", 1, NULL, 'P'}, @@ -148,7 +149,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "acCd:ef:F:h:I:j:lL:n:Op:P:RsS:t:T:U:vwWx1", + while ((c = getopt_long(argc, argv, "acCd:ef:F:h:I:j:lL:n:N:Op:P:RsS:t:T:U:vwWx1", cmdopts, NULL)) != -1) { switch (c) @@ -196,6 +197,10 @@ main(int argc, char **argv) simple_string_list_append(&opts->schemaNames, optarg); break; + case 'N': /* Do not dump data for this schema */ + simple_string_list_append(&opts->schemaExcludeNames, optarg); + break; + case 'O': opts->noOwner = 1; break; @@ -456,6 +461,7 @@ usage(const char *progname) printf(_(" -L, --use-list=FILENAME use table of contents from this file for\n" " selecting/ordering output\n")); printf(_(" -n, --schema=NAME restore only objects in this schema\n")); + printf(_(" -N, --exclude-schema=NAME do not restore objects in this schema\n")); printf(_(" -O, --no-owner skip restoration of object ownership\n")); printf(_(" -P, --function=NAME(args) restore named function\n")); printf(_(" -s, --schema-only restore only the schema, no data\n")); From 65c65563842cc99fb1c349211581a62dc728eee2 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 21 Sep 2016 13:14:48 +0300 Subject: [PATCH 193/871] Fix pgbench's calculation of average latency, when -T is not used. If the test duration was given in # of transactions (-t or no option), rather as a duration (-T), the latency average was always printed as 0. It has been broken ever since the display of latency average was added, in 9.4. Fabien Coelho Discussion: --- src/bin/pgbench/pgbench.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 4676a59020..9033ff2caa 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -3296,9 +3296,11 @@ printResults(TState *threads, StatsData *total, instr_time total_time, if (throttle_delay || progress || latency_limit) printSimpleStats("latency", &total->latency); else - /* only an average latency computed from the duration is available */ + { + /* no measurement, show average latency computed from run time */ printf("latency average: %.3f ms\n", - 1000.0 * duration * nclients / total->cnt); + 1000.0 * time_include * nclients / total->cnt); + } if (throttle_delay) { From 2a7f4f76434d82eb0d1b5f4f7051043e1dd3ee1a Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 21 Sep 2016 13:24:13 +0300 Subject: [PATCH 194/871] Print test parameters like "foo: 123", and results like "foo = 123". The way "latency average" was printed was differently if it was calculated from the overall run time or was measured on a per-transaction basis. Also, the per-script weight is a test parameter, rather than a result, so use the "weight: %f" style for that. Backpatch to 9.6, since the inconsistency on "latency average" was introduced there. Fabien Coelho Discussion: --- src/bin/pgbench/pgbench.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 9033ff2caa..8b24ad50e7 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -3260,6 +3260,7 @@ printResults(TState *threads, StatsData *total, instr_time total_time, tps_exclude = total->cnt / (time_include - (INSTR_TIME_GET_DOUBLE(conn_total_time) / nclients)); + /* Report test parameters. */ printf("transaction type: %s\n", num_scripts == 1 ? sql_script[0].desc : "multiple scripts"); printf("scaling factor: %d\n", scale); @@ -3298,7 +3299,7 @@ printResults(TState *threads, StatsData *total, instr_time total_time, else { /* no measurement, show average latency computed from run time */ - printf("latency average: %.3f ms\n", + printf("latency average = %.3f ms\n", 1000.0 * time_include * nclients / total->cnt); } @@ -3326,7 +3327,7 @@ printResults(TState *threads, StatsData *total, instr_time total_time, { if (num_scripts > 1) printf("SQL script %d: %s\n" - " - weight = %d (targets %.1f%% of total)\n" + " - weight: %d (targets %.1f%% of total)\n" " - " INT64_FORMAT " transactions (%.1f%% of total, tps = %f)\n", i + 1, sql_script[i].desc, sql_script[i].weight, From 60270e5e00850ee8cc34296e38d0000415c8b152 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 21 Sep 2016 08:37:02 -0400 Subject: [PATCH 195/871] Add more parallel query documentation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, the individual settings were documented, but there was no overall discussion of the capabilities and limitations of the feature. Add that. Patch by me, reviewed by Peter Eisentraut and Álvaro Herrera. --- doc/src/sgml/config.sgml | 5 + doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/parallel.sgml | 472 +++++++++++++++++++++++++++++++++++++ doc/src/sgml/postgres.sgml | 1 + 4 files changed, 479 insertions(+) create mode 100644 doc/src/sgml/parallel.sgml diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index cd66abc8ba..a848a7edd1 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2027,6 +2027,11 @@ include_dir 'conf.d' as much CPU time, memory, I/O bandwidth, and so forth as a query which uses no workers at all. + + + For more information on parallel query, see + . + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 43837114ba..69649a7da4 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -24,6 +24,7 @@ + diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml new file mode 100644 index 0000000000..c80d42dbef --- /dev/null +++ b/doc/src/sgml/parallel.sgml @@ -0,0 +1,472 @@ + + + + Parallel Query + + + parallel query + + + + PostgreSQL can devise query plans which can leverage + multiple CPUs in order to answer queries faster. This feature is known + as parallel query. Many queries cannot benefit from parallel query, either + due to limitations of the current implementation or because there is no + imaginable query plan which is any faster than the serial query plan. + However, for queries that can benefit, the speedup from parallel query + is often very significant. Many queries can run more than twice as fast + when using parallel query, and some queries can run four times faster or + even more. Queries that touch a large amount of data but return only a + few rows to the user will typically benefit most. This chapter explains + some details of how parallel query works and in which situations it can be + used so that users who wish to make use of it can understand what to expect. + + + + How Parallel Query Works + + + When the optimizer determines that parallel query is the fastest execution + strategy for a particular query, it will create a query plan which includes + a Gather node. Here is a simple example: + + +EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; + QUERY PLAN +------------------------------------------------------------------------------------- + Gather (cost=1000.00..217018.43 rows=1 width=97) + Workers Planned: 2 + -> Parallel Seq Scan on pgbench_accounts (cost=0.00..216018.33 rows=1 width=97) + Filter: (filler ~~ '%x%'::text) +(4 rows) + + + + + In all cases, the Gather node will have exactly one + child plan, which is the portion of the plan that will be executed in + parallel. If the Gather node is at the very top of the plan + tree, then the entire query will execute in parallel. If it is somewhere + else in the plan tree, then only that portion of the query will run in + parallel. In the example above, the query accesses only one table, so + there is only one plan node other than the Gather node itself; + since that plan node is a child of the Gather node, it will + run in parallel. + + + + Using EXPLAIN, you can see the number of + workers chosen by the planner. When the Gather node is reached + during query execution, the process which is implementing the user's + session will request a number of background + worker processes equal to the number + of workers chosen by the planner. The total number of background + workers that can exist at any one time is limited by + , so it is possible for a + parallel query to run with fewer workers than planned, or even with + no workers at all. The optimal plan may depend on the number of workers + that are available, so this can result in poor query performance. If this + occurrence is frequent, considering increasing + max_worker_processes so that more workers can be run + simultaneously or alternatively reducing + so that the planner + requests fewer workers. + + + + Every background worker process which is successfully started for a given + parallel query will execute the portion of the plan which is a descendent + of the Gather node. The leader will also execute that portion + of the plan, but it has an additional responsibility: it must also read + all of the tuples generated by the workers. When the parallel portion of + the plan generates only a small number of tuples, the leader will often + behave very much like an additional worker, speeding up query execution. + Conversely, when the parallel portion of the plan generates a large number + of tuples, the leader may be almost entirely occupied with reading the + tuples generated by the workers and performing any further processing + steps which are required by plan nodes above the level of the + Gather node. In such cases, the leader will do very + little of the work of executing the parallel portion of the plan. + + + + + When Can Parallel Query Be Used? + + + There are several settings which can cause the query planner not to + generate a parallel query plan under any circumstances. In order for + any parallel query plans whatsoever to be generated, the following + settings must be configured as indicated. + + + + + + must be set to a + value which is greater than zero. This is a special case of the more + general principle that no more workers should be used than the number + configured via max_parallel_workers_per_gather. + + + + + + must be set to a + value other than none. Parallel query requires dynamic + shared memory in order to pass data between cooperating processes. + + + + + + In addition, the system must not be running in single-user mode. Since + the entire database system is running in single process in this situation, + no background workers will be available. + + + + Even when it is in general possible for parallel query plans to be + generated, the planner will not generate them for a given query + if any of the following are true: + + + + + + The query writes any data or locks any database rows. If a query + contains a data-modifying operation either at the top level or within + a CTE, no parallel plans for that query will be generated. This is a + limitation of the current implementation which could be lifted in a + future release. + + + + + + The query might be suspended during execution. In any situation in + which the system thinks that partial or incremental execution might + occur, no parallel plan is generated. For example, a cursor created + using DECLARE CURSOR will never use + a parallel plan. Similarly, a PL/pgsql loop of the form + FOR x IN query LOOP .. END LOOP will never use a + parallel plan, because the parallel query system is unable to verify + that the code in the loop is safe to execute while parallel query is + active. + + + + + + The query uses any function marked PARALLEL UNSAFE. + Most system-defined functions are PARALLEL SAFE, + but user-defined functions are marked PARALLEL + UNSAFE by default. See the discussion of + . + + + + + + The query is running inside of another query that is already parallel. + For example, if a function called by a parallel query issues an SQL + query itself, that query will never use a parallel plan. This is a + limitation of the current implementation, but it may not be desirable + to remove this limitation, since it could result in a single query + using a very large number of processes. + + + + + + The transaction isolation level is serializable. This is + a limitation of the current implementation. + + + + + + Even when parallel query plan is generated for a particular query, there + are several circumstances under which it will be impossible to execute + that plan in parallel at execution time. If this occurs, the leader + will execute the portion of the plan between below the Gather + node entirely by itself, almost as if the Gather node were + not present. This will happen if any of the following conditions are met: + + + + + + No background workers can be obtained because of the limitation that + the total number of background workers cannot exceed + . + + + + + + The client sends an Execute message with a non-zero fetch count. + See the discussion of the + extended query protocol. + Since libpq currently provides no way to + send such a message, this can only occur when using a client that + does not rely on libpq. If this is a frequent + occurrence, it may be a good idea to set + in sessions + where it is likely, so as to avoid generating query plans that may + be suboptimal when run serially. + + + + + + The transaction isolation level is serializable. This situation + does not normally arise, because parallel query plans are not + generated when the transaction isolation level is serializable. + However, it can happen if the transaction isolation level is changed to + serializable after the plan is generated and before it is executed. + + + + + + + Parallel Plans + + + Because each worker executes the parallel portion of the plan to + completion, it is not possible to simply take an ordinary query plan + and run it using multiple workers. Each worker would produce a full + copy of the output result set, so the query would not run any faster + than normal but would produce incorrect results. Instead, the parallel + portion of the plan must be what is known internally to the query + optimizer as a partial plan; that is, it must constructed + so that each process will which executes the plan will generate only a + subset of the output rows in such a way that each required output row + is guaranteed to be generated by exactly one of the cooperating processes. + + + + Parallel Scans + + + Currently, the only type of scan which has been modified to work with + parallel query is a sequential scan. Therefore, the driving table in + a parallel plan will always be scanned using a + Parallel Seq Scan. The relation's blocks will be divided + among the cooperating processes. Blocks are handed out one at a + time, so that access to the relation remains sequential. Each process + will visit every tuple on the page assigned to it before requesting a new + page. + + + + + Parallel Joins + + + The driving table may be joined to one or more other tables using nested + loops or hash joins. The outer side of the join may be any kind of + non-parallel plan that is otherwise supported by the planner provided that + it is safe to run within a parallel worker. For example, it may be an + index scan which looks up a value based on a column taken from the inner + table. Each worker will execute the outer side of the plan in full, which + is why merge joins are not supported here. The outer side of a merge join + will often involve sorting the entire inner table; even if it involves an + index, it is unlikely to be productive to have multiple processes each + conduct a full index scan of the inner table. + + + + + Parallel Aggregation + + It is not possible to perform the aggregation portion of a query entirely + in parallel. For example, if a query involves selecting + COUNT(*), each worker could compute a total, but those totals + would need to combined in order to produce a final answer. If the query + involved a GROUP BY clause, a separate total would need to + be computed for each group. Even though aggregation can't be done entirely + in parallel, queries involving aggregation are often excellent candidates + for parallel query, because they typically read many rows but return only + a few rows to the client. Queries that return many rows to the client + are often limited by the speed at which the client can read the data, + in which case parallel query cannot help very much. + + + + PostgreSQL supports parallel aggregation by aggregating + twice. First, each process participating in the parallel portion of the + query performs an aggregation step, producing a partial result for each + group of which that process is aware. This is reflected in the plan as + a PartialAggregate node. Second, the partial results are + transferred to the leader via the Gather node. Finally, the + leader re-aggregates the results across all workers in order to produce + the final result. This is reflected in the plan as a + FinalizeAggregate node. + + + + Parallel aggregation is not supported in all situations. Each aggregate + must be safe for parallelism and must + have a combine function. If the aggregate has a transition state of type + internal, it must have serialization and deserialization + functions. See for more details. + Parallel aggregation is not supported for ordered set aggregates or when + the query involves GROUPING SETS. It can only be used when + all joins involved in the query are also part of the parallel portion + of the plan. + + + + + + Parallel Plan Tips + + + If a query that is expected to do so does not produce a parallel plan, + you can try reducing or + . Of course, this plan may turn + out to be slower than the serial plan which the planner preferred, but + this will not always be the case. If you don't get a parallel + plan even with very small values of these settings (e.g. after setting + them both to zero), there may be some reason why the query planner is + unable to generate a parallel plan for your query. See + and + for information on why this may be + the case. + + + + When executing a parallel plan, you can use EXPLAIN (ANALYZE, + VERBOSE) will display per-worker statistics for each plan node. + This may be useful in determining whether the work is being evenly + distributed between all plan nodes and more generally in understanding the + performance characteristics of the plan. + + + + + + + Parallel Safety + + + The planner classifies operations involved in a query as either + parallel safe, parallel restricted, + or parallel unsafe. A parallel safe operation is one which + does not conflict with the use of parallel query. A parallel restricted + operation is one which cannot be performed in a parallel worker, but which + can be performed in the leader while parallel query is in use. Therefore, + parallel restricted operations can never occur below a Gather + node, but can occur elsewhere in a plan which contains a + Gather node. A parallel unsafe operation is one which cannot + be performed while parallel query is in use, not even in the leader. + When a query contains anything which is parallel unsafe, parallel query + is completely disabled for that query. + + + + The following operations are always parallel restricted. + + + + + + Scans of common table expressions (CTEs). + + + + + + Scans of temporary tables. + + + + + + Scans of foreign tables, unless the foreign data wrapper has + an IsForeignScanParallelSafe API which indicates otherwise. + + + + + + Access to an InitPlan or SubPlan. + + + + + + Parallel Labeling for Functions and Aggregates + + + The planner cannot automatically determine whether a user-defined + function or aggregate is parallel safe, parallel restricted, or parallel + unsafe, because this would require predicting every operation which the + function could possibly perform. In general, this is equivalent to the + Halting Problem and therefore impossible. Even for simple functions + where it conceivably be done, we do not try, since this would be expensive + and error-prone. Instead, all user-defined functions are assumed to + be parallel unsafe unless otherwise marked. When using + or + , markings can be set by specifying + PARALLEL SAFE, PARALLEL RESTRICTED, or + PARALLEL UNSAFE as appropriate. When using + , the + PARALLEL option can be specified with SAFE, + RESTRICTED, or UNSAFE as the corresponding value. + + + + Functions and aggregates must be marked PARALLEL UNSAFE if + they write to the database, access sequences, change the transaction state + even temporarily (e.g. a PL/pgsql function which establishes an + EXCEPTION block to catch errors), or make persistent changes to + settings. Similarly, functions must be marked PARALLEL + RESTRICTED if they access temporary tables, client connection state, + cursors, prepared statements, or miscellaneous backend-local state which + the system cannot synchronize across workers. For example, + setseed and random are parallel restricted for + this last reason. + + + + In general, if a function is labeled as being safe when it is restricted or + unsafe, or if it is labeled as being restricted when it is in fact unsafe, + it may throw errors or produce wrong answers when used in a parallel query. + C-language functions could in theory exhibit totally undefined behavior if + mislabeled, since there is no way for the system to protect itself against + arbitrary C code, but in most likely cases the result will be no worse than + for any other function. If in doubt, it is probably best to label functions + as UNSAFE. + + + + If a function executed within a parallel worker acquires locks which are + not held by the leader, for example by querying a table not referenced in + the query, those locks will be released at worker exit, not end of + transaction. If you write a function which does this, and this behavior + difference is important to you, mark such functions as + PARALLEL RESTRICTED + to ensure that they execute only in the leader. + + + + Note that the query planner does not consider deferring the evaluation of + parallel-restricted functions or aggregates involved in the query in + order to obtain a superior plan. So, for example, if a WHERE + clause applied to a particular table is parallel restricted, the query + planner will not consider placing the scan of that table below a + Gather node. In some cases, it would be + possible (and perhaps even efficient) to include the scan of that table in + the parallel portion of the query and defer the evaluation of the + WHERE clause so that it happens above the Gather + node. However, the planner does not do this. + + + + + + + diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 0346d367e5..9143917c49 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -106,6 +106,7 @@ &textsearch; &mvcc; &perform; + ∥ From c91b34bab19bdc4fca4de37cd37c8a4e79dae4d0 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 21 Sep 2016 12:00:00 -0400 Subject: [PATCH 196/871] Fix typo From: Michael Paquier --- src/include/pgstat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/pgstat.h b/src/include/pgstat.h index dc3320d091..0c98c59e72 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -1007,9 +1007,9 @@ extern void pgstat_initstats(Relation rel); * * Called from places where server process needs to wait. This is called * to report wait event information. The wait information is stored - * as 4-bytes where first byte repersents the wait event class (type of + * as 4-bytes where first byte represents the wait event class (type of * wait, for different types of wait, refer WaitClass) and the next - * 3-bytes repersent the actual wait event. Currently 2-bytes are used + * 3-bytes represent the actual wait event. Currently 2-bytes are used * for wait event which is sufficient for current usage, 1-byte is * reserved for future usage. * From e767db2242079b4ec2a7bbd18da7649d9ee1d696 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 26 Jul 2016 11:39:43 -0400 Subject: [PATCH 197/871] Make command_like output more compact Consistently print the test name, not the full command, which can be quite lenghty and include temporary directory names and other distracting details. Reviewed-by: Michael Paquier --- src/test/perl/TestLib.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/perl/TestLib.pm b/src/test/perl/TestLib.pm index 649fd82173..c7b3262087 100644 --- a/src/test/perl/TestLib.pm +++ b/src/test/perl/TestLib.pm @@ -276,8 +276,8 @@ sub command_like my ($stdout, $stderr); print("# Running: " . join(" ", @{$cmd}) . "\n"); my $result = IPC::Run::run $cmd, '>', \$stdout, '2>', \$stderr; - ok($result, "@$cmd exit code 0"); - is($stderr, '', "@$cmd no stderr"); + ok($result, "$test_name: exit code 0"); + is($stderr, '', "$test_name: no stderr"); like($stdout, $expected_stdout, "$test_name: matches"); } From eb5089a05ba0852cc3eafea53c5d23e7633fca81 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 26 Jul 2016 10:48:43 -0400 Subject: [PATCH 198/871] pg_ctl: Add tests for promote action Reviewed-by: Michael Paquier --- src/bin/pg_ctl/t/003_promote.pl | 39 +++++++++++++++++++++++++++++++++ src/test/perl/TestLib.pm | 11 ++++++++++ 2 files changed, 50 insertions(+) create mode 100644 src/bin/pg_ctl/t/003_promote.pl diff --git a/src/bin/pg_ctl/t/003_promote.pl b/src/bin/pg_ctl/t/003_promote.pl new file mode 100644 index 0000000000..1461234f2a --- /dev/null +++ b/src/bin/pg_ctl/t/003_promote.pl @@ -0,0 +1,39 @@ +use strict; +use warnings; + +use PostgresNode; +use TestLib; +use Test::More tests => 9; + +my $tempdir = TestLib::tempdir; + +command_fails_like([ 'pg_ctl', '-D', "$tempdir/nonexistent", 'promote' ], + qr/directory .* does not exist/, + 'pg_ctl promote with nonexistent directory'); + +my $node_primary = get_new_node('primary'); +$node_primary->init(allows_streaming => 1); + +command_fails_like([ 'pg_ctl', '-D', $node_primary->data_dir, 'promote' ], + qr/PID file .* does not exist/, + 'pg_ctl promote of not running instance fails'); + +$node_primary->start; + +command_fails_like([ 'pg_ctl', '-D', $node_primary->data_dir, 'promote' ], + qr/not in standby mode/, + 'pg_ctl promote of primary instance fails'); + +my $node_standby = get_new_node('standby'); +$node_primary->backup('my_backup'); +$node_standby->init_from_backup($node_primary, 'my_backup', has_streaming => 1); +$node_standby->start; + +is($node_standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), + 't', 'standby is in recovery'); + +command_ok([ 'pg_ctl', '-D', $node_standby->data_dir, 'promote' ], + 'pg_ctl promote of standby runs'); + +ok($node_standby->poll_query_until('postgres', 'SELECT NOT pg_is_in_recovery()'), + 'promoted standby is not in recovery'); diff --git a/src/test/perl/TestLib.pm b/src/test/perl/TestLib.pm index c7b3262087..51b533e08c 100644 --- a/src/test/perl/TestLib.pm +++ b/src/test/perl/TestLib.pm @@ -34,6 +34,7 @@ our @EXPORT = qw( program_version_ok program_options_handling_ok command_like + command_fails_like $windows_os ); @@ -281,4 +282,14 @@ sub command_like like($stdout, $expected_stdout, "$test_name: matches"); } +sub command_fails_like +{ + my ($cmd, $expected_stderr, $test_name) = @_; + my ($stdout, $stderr); + print("# Running: " . join(" ", @{$cmd}) . "\n"); + my $result = IPC::Run::run $cmd, '>', \$stdout, '2>', \$stderr; + ok(!$result, "$test_name: exit code not 0"); + like($stderr, $expected_stderr, "$test_name: matches"); +} + 1; From c1dc51d4844e2a37412b034c07c1c5a439ba0b9d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 26 Jul 2016 11:23:43 -0400 Subject: [PATCH 199/871] pg_ctl: Detect current standby state from pg_control pg_ctl used to determine whether a server was in standby mode by looking for a recovery.conf file. With this change, it instead looks into pg_control, which is potentially more accurate. There are also occasional discussions about removing recovery.conf, so this removes one dependency. Reviewed-by: Michael Paquier --- src/backend/utils/misc/pg_controldata.c | 12 +++++++ src/bin/pg_controldata/pg_controldata.c | 4 +++ src/bin/pg_ctl/pg_ctl.c | 47 ++++++++++++++++++++----- src/common/controldata_utils.c | 15 ++++---- src/include/common/controldata_utils.h | 2 +- 5 files changed, 62 insertions(+), 18 deletions(-) diff --git a/src/backend/utils/misc/pg_controldata.c b/src/backend/utils/misc/pg_controldata.c index 34ee76a237..4f9de83097 100644 --- a/src/backend/utils/misc/pg_controldata.c +++ b/src/backend/utils/misc/pg_controldata.c @@ -52,6 +52,9 @@ pg_control_system(PG_FUNCTION_ARGS) /* read the control file */ ControlFile = get_controlfile(DataDir, NULL); + if (!ControlFile) + ereport(ERROR, + (errmsg("calculated CRC checksum does not match value stored in file"))); values[0] = Int32GetDatum(ControlFile->pg_control_version); nulls[0] = false; @@ -128,6 +131,9 @@ pg_control_checkpoint(PG_FUNCTION_ARGS) /* Read the control file. */ ControlFile = get_controlfile(DataDir, NULL); + if (!ControlFile) + ereport(ERROR, + (errmsg("calculated CRC checksum does not match value stored in file"))); /* * Calculate name of the WAL file containing the latest checkpoint's REDO @@ -230,6 +236,9 @@ pg_control_recovery(PG_FUNCTION_ARGS) /* read the control file */ ControlFile = get_controlfile(DataDir, NULL); + if (!ControlFile) + ereport(ERROR, + (errmsg("calculated CRC checksum does not match value stored in file"))); values[0] = LSNGetDatum(ControlFile->minRecoveryPoint); nulls[0] = false; @@ -295,6 +304,9 @@ pg_control_init(PG_FUNCTION_ARGS) /* read the control file */ ControlFile = get_controlfile(DataDir, NULL); + if (!ControlFile) + ereport(ERROR, + (errmsg("calculated CRC checksum does not match value stored in file"))); values[0] = Int32GetDatum(ControlFile->maxAlign); nulls[0] = false; diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index 96619a2076..e92feabade 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -156,6 +156,10 @@ main(int argc, char *argv[]) /* get a copy of the control file */ ControlFile = get_controlfile(DataDir, progname); + if (!ControlFile) + printf(_("WARNING: Calculated CRC checksum does not match value stored in file.\n" + "Either the file is corrupt, or it has a different layout than this program\n" + "is expecting. The results below are untrustworthy.\n\n")); /* * This slightly-chintzy coding will work as long as the control file diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index efc07291ad..eb8a67a903 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -19,6 +19,8 @@ #include "postgres_fe.h" +#include "catalog/pg_control.h" +#include "common/controldata_utils.h" #include "libpq-fe.h" #include "pqexpbuffer.h" @@ -96,7 +98,6 @@ static char postopts_file[MAXPGPATH]; static char version_file[MAXPGPATH]; static char pid_file[MAXPGPATH]; static char backup_file[MAXPGPATH]; -static char recovery_file[MAXPGPATH]; static char promote_file[MAXPGPATH]; #ifdef WIN32 @@ -158,6 +159,8 @@ static bool postmaster_is_alive(pid_t pid); static void unlimit_core_size(void); #endif +static DBState get_control_dbstate(void); + #ifdef WIN32 static void @@ -988,12 +991,12 @@ do_stop(void) /* * If backup_label exists, an online backup is running. Warn the user * that smart shutdown will wait for it to finish. However, if - * recovery.conf is also present, we're recovering from an online + * the server is in archive recovery, we're recovering from an online * backup instead of performing one. */ if (shutdown_mode == SMART_MODE && stat(backup_file, &statbuf) == 0 && - stat(recovery_file, &statbuf) != 0) + get_control_dbstate() != DB_IN_ARCHIVE_RECOVERY) { print_msg(_("WARNING: online backup mode is active\n" "Shutdown will not complete until pg_stop_backup() is called.\n\n")); @@ -1076,12 +1079,12 @@ do_restart(void) /* * If backup_label exists, an online backup is running. Warn the user * that smart shutdown will wait for it to finish. However, if - * recovery.conf is also present, we're recovering from an online + * the server is in archive recovery, we're recovering from an online * backup instead of performing one. */ if (shutdown_mode == SMART_MODE && stat(backup_file, &statbuf) == 0 && - stat(recovery_file, &statbuf) != 0) + get_control_dbstate() != DB_IN_ARCHIVE_RECOVERY) { print_msg(_("WARNING: online backup mode is active\n" "Shutdown will not complete until pg_stop_backup() is called.\n\n")); @@ -1168,7 +1171,6 @@ do_promote(void) { FILE *prmfile; pgpid_t pid; - struct stat statbuf; pid = get_pgpid(false); @@ -1187,8 +1189,7 @@ do_promote(void) exit(1); } - /* If recovery.conf doesn't exist, the server is not in standby mode */ - if (stat(recovery_file, &statbuf) != 0) + if (get_control_dbstate() != DB_IN_ARCHIVE_RECOVERY) { write_stderr(_("%s: cannot promote server; " "server is not in standby mode\n"), @@ -2115,6 +2116,35 @@ adjust_data_dir(void) } +static DBState +get_control_dbstate(void) +{ + DBState ret; + + for (;;) + { + ControlFileData *control_file_data = get_controlfile(pg_data, progname); + + if (control_file_data) + { + ret = control_file_data->state; + pfree(control_file_data); + return ret; + } + + if (wait_seconds > 0) + { + pg_usleep(1000000); /* 1 sec */ + wait_seconds--; + continue; + } + + write_stderr(_("%s: control file appears to be corrupt\n"), progname); + exit(1); + } +} + + int main(int argc, char **argv) { @@ -2401,7 +2431,6 @@ main(int argc, char **argv) snprintf(version_file, MAXPGPATH, "%s/PG_VERSION", pg_data); snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pg_data); snprintf(backup_file, MAXPGPATH, "%s/backup_label", pg_data); - snprintf(recovery_file, MAXPGPATH, "%s/recovery.conf", pg_data); } switch (ctl_command) diff --git a/src/common/controldata_utils.c b/src/common/controldata_utils.c index 5592fe7039..f218d2558c 100644 --- a/src/common/controldata_utils.c +++ b/src/common/controldata_utils.c @@ -33,9 +33,11 @@ * * Get controlfile values. The caller is responsible * for pfreeing the result. + * + * Returns NULL if the CRC did not match. */ ControlFileData * -get_controlfile(char *DataDir, const char *progname) +get_controlfile(const char *DataDir, const char *progname) { ControlFileData *ControlFile; int fd; @@ -82,13 +84,10 @@ get_controlfile(char *DataDir, const char *progname) FIN_CRC32C(crc); if (!EQ_CRC32C(crc, ControlFile->crc)) -#ifndef FRONTEND - elog(ERROR, _("calculated CRC checksum does not match value stored in file")); -#else - printf(_("WARNING: Calculated CRC checksum does not match value stored in file.\n" - "Either the file is corrupt, or it has a different layout than this program\n" - "is expecting. The results below are untrustworthy.\n\n")); -#endif + { + pfree(ControlFile); + return NULL; + } /* Make sure the control file is valid byte order. */ if (ControlFile->pg_control_version % 65536 == 0 && diff --git a/src/include/common/controldata_utils.h b/src/include/common/controldata_utils.h index a355d2252d..f834624e4e 100644 --- a/src/include/common/controldata_utils.h +++ b/src/include/common/controldata_utils.h @@ -12,6 +12,6 @@ #include "catalog/pg_control.h" -extern ControlFileData *get_controlfile(char *DataDir, const char *progname); +extern ControlFileData *get_controlfile(const char *DataDir, const char *progname); #endif /* COMMON_CONTROLDATA_UTILS_H */ From ebdf5bf7d1c97a926e2b0cb6523344c2643623c7 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 26 Jul 2016 12:08:49 -0400 Subject: [PATCH 200/871] Delay updating control file to "in production" Move the updating of the control file to "in production" status until the point where WAL writes are allowed. Before, there could be a significant gap between the control file update and write transactions actually being allowed. This makes it more reliable to use the control status to verify the end of a promotion. From: Michael Paquier --- src/backend/access/transam/xlog.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 2189c22c64..c1b9a97147 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -7431,12 +7431,6 @@ StartupXLOG(void) */ InRecovery = false; - LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); - ControlFile->state = DB_IN_PRODUCTION; - ControlFile->time = (pg_time_t) time(NULL); - UpdateControlFile(); - LWLockRelease(ControlFileLock); - /* start the archive_timeout timer running */ XLogCtl->lastSegSwitchTime = (pg_time_t) time(NULL); @@ -7494,15 +7488,32 @@ StartupXLOG(void) CompleteCommitTsInitialization(); /* - * All done. Allow backends to write WAL. (Although the bool flag is - * probably atomic in itself, we use the info_lck here to ensure that - * there are no race conditions concerning visibility of other recent - * updates to shared memory.) + * All done with end-of-recovery actions. + * + * Now allow backends to write WAL and update the control file status in + * consequence. The boolean flag allowing backends to write WAL is + * updated while holding ControlFileLock to prevent other backends to look + * at an inconsistent state of the control file in shared memory. There + * is still a small window during which backends can write WAL and the + * control file is still referring to a system not in DB_IN_PRODUCTION + * state while looking at the on-disk control file. + * + * Also, although the boolean flag to allow WAL is probably atomic in + * itself, we use the info_lck here to ensure that there are no race + * conditions concerning visibility of other recent updates to shared + * memory. */ + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); + ControlFile->state = DB_IN_PRODUCTION; + ControlFile->time = (pg_time_t) time(NULL); + SpinLockAcquire(&XLogCtl->info_lck); XLogCtl->SharedRecoveryInProgress = false; SpinLockRelease(&XLogCtl->info_lck); + UpdateControlFile(); + LWLockRelease(ControlFileLock); + /* * If there were cascading standby servers connected to us, nudge any wal * sender processes to notice that we've been promoted. From e7010ce4794a4c12a6a8bfb0ca1de49b61046847 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 5 Aug 2016 21:35:19 -0400 Subject: [PATCH 201/871] pg_ctl: Add wait option to promote action When waiting is selected for the promote action, look into pg_control until the state changes, then use the PQping-based waiting until the server is reachable. Reviewed-by: Michael Paquier --- doc/src/sgml/ref/pg_ctl-ref.sgml | 29 +++++++++++++++----- src/bin/pg_ctl/pg_ctl.c | 45 +++++++++++++++++++++++--------- src/bin/pg_ctl/t/003_promote.pl | 18 ++++++++++++- 3 files changed, 72 insertions(+), 20 deletions(-) diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index 6ceb7816dc..a00c355f4a 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -91,6 +91,8 @@ PostgreSQL documentation pg_ctl + + seconds datadir @@ -361,8 +363,8 @@ PostgreSQL documentation - The maximum number of seconds to wait when waiting for startup or - shutdown to complete. Defaults to the value of the + The maximum number of seconds to wait when waiting for an operation + to complete (see option ). Defaults to the value of the PGCTLTIMEOUT environment variable or, if not set, to 60 seconds. @@ -383,8 +385,23 @@ PostgreSQL documentation - Wait for the startup or shutdown to complete. - Waiting is the default option for shutdowns, but not startups. + Wait for an operation to complete. This is supported for the + modes start, stop, + restart, promote, + and register. + + + + Waiting is the default option for shutdowns, but not startups, + restarts, or promotions. This is mainly for historical reasons; the + waiting option is almost always preferable. If waiting is not + selected, the requested action is triggered, but there is no feedback + about its success. In that case, the server log file or an external + monitoring system would have to be used to check the progress and + success of the operation. + + + When waiting for startup, pg_ctl repeatedly attempts to connect to the server. When waiting for shutdown, pg_ctl waits for @@ -400,8 +417,8 @@ PostgreSQL documentation - Do not wait for startup or shutdown to complete. This is the - default for start and restart modes. + Do not wait for an operation to complete. This is the opposite of the + option . diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index eb8a67a903..2f0976a9cc 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -1228,7 +1228,34 @@ do_promote(void) exit(1); } - print_msg(_("server promoting\n")); + if (do_wait) + { + DBState state = DB_STARTUP; + + print_msg(_("waiting for server to promote...")); + while (wait_seconds > 0) + { + state = get_control_dbstate(); + if (state == DB_IN_PRODUCTION) + break; + + print_msg("."); + pg_usleep(1000000); /* 1 sec */ + wait_seconds--; + } + if (state == DB_IN_PRODUCTION) + { + print_msg(_(" done\n")); + print_msg(_("server promoted\n")); + } + else + { + print_msg(_(" stopped waiting\n")); + print_msg(_("server is still promoting\n")); + } + } + else + print_msg(_("server promoting\n")); } @@ -2405,18 +2432,10 @@ main(int argc, char **argv) if (!wait_set) { - switch (ctl_command) - { - case RESTART_COMMAND: - case START_COMMAND: - do_wait = false; - break; - case STOP_COMMAND: - do_wait = true; - break; - default: - break; - } + if (ctl_command == STOP_COMMAND) + do_wait = true; + else + do_wait = false; } if (ctl_command == RELOAD_COMMAND) diff --git a/src/bin/pg_ctl/t/003_promote.pl b/src/bin/pg_ctl/t/003_promote.pl index 1461234f2a..0b6090b6eb 100644 --- a/src/bin/pg_ctl/t/003_promote.pl +++ b/src/bin/pg_ctl/t/003_promote.pl @@ -3,7 +3,7 @@ use PostgresNode; use TestLib; -use Test::More tests => 9; +use Test::More tests => 12; my $tempdir = TestLib::tempdir; @@ -37,3 +37,19 @@ ok($node_standby->poll_query_until('postgres', 'SELECT NOT pg_is_in_recovery()'), 'promoted standby is not in recovery'); + +# same again with wait option +$node_standby = get_new_node('standby2'); +$node_standby->init_from_backup($node_primary, 'my_backup', has_streaming => 1); +$node_standby->start; + +is($node_standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), + 't', 'standby is in recovery'); + +command_ok([ 'pg_ctl', '-D', $node_standby->data_dir, '-w', 'promote' ], + 'pg_ctl -w promote of standby runs'); + +# no wait here + +is($node_standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), + 'f', 'promoted standby is not in recovery'); From 8b845520fb0aa50fea7aae44a45cee1b6d87845d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 4 Aug 2016 14:44:23 -0400 Subject: [PATCH 202/871] Add tests for various connection string issues Add tests for consistent support of connection strings in frontend programs as well as proper handling of unusual characters in database and user names. These tests were developed for the issues of CVE-2016-5424. To allow testing of names with spaces, change the pg_regress command-line options --create-role and --dbname to split their arguments by comma only, not space or comma as before. Only commas were actually used in existing uses. Noah Misch, Michael Paquier, Peter Eisentraut --- src/bin/pg_dump/t/010_dump_connstr.pl | 142 ++++++++++++++++++++++++++ src/bin/pg_rewind/RewindTest.pm | 2 +- src/bin/scripts/t/010_clusterdb.pl | 5 +- src/bin/scripts/t/090_reindexdb.pl | 9 +- src/bin/scripts/t/100_vacuumdb.pl | 4 +- src/bin/scripts/t/200_connstr.pl | 38 +++++++ src/test/perl/PostgresNode.pm | 29 +++++- src/test/perl/TestLib.pm | 14 +++ src/test/regress/pg_regress.c | 35 +++++-- 9 files changed, 266 insertions(+), 12 deletions(-) create mode 100644 src/bin/pg_dump/t/010_dump_connstr.pl create mode 100644 src/bin/scripts/t/200_connstr.pl diff --git a/src/bin/pg_dump/t/010_dump_connstr.pl b/src/bin/pg_dump/t/010_dump_connstr.pl new file mode 100644 index 0000000000..2d0d1e4298 --- /dev/null +++ b/src/bin/pg_dump/t/010_dump_connstr.pl @@ -0,0 +1,142 @@ +use strict; +use warnings; + +use PostgresNode; +use TestLib; +use Test::More tests => 14; + +# In a SQL_ASCII database, pgwin32_message_to_UTF16() needs to +# interpret everything as UTF8. We're going to use byte sequences +# that aren't valid UTF-8 strings, so that would fail. Use LATIN1, +# which accepts any byte and has a conversion from each byte to UTF-8. +$ENV{LC_ALL} = 'C'; +$ENV{PGCLIENTENCODING} = 'LATIN1'; + +# Create database and user names covering the range of LATIN1 +# characters, for use in a connection string by pg_dumpall. Skip ',' +# because of pg_regress --create-role, skip [\n\r] because pg_dumpall +# does not allow them. +my $dbname1 = generate_ascii_string(1, 9) . + generate_ascii_string(11, 12) . + generate_ascii_string(14, 33) . + ($TestLib::windows_os ? '' : '"x"') . # IPC::Run mishandles '"' on Windows + generate_ascii_string(35, 43) . + generate_ascii_string(45, 63); # contains '=' +my $dbname2 = generate_ascii_string(67, 129); # skip 64-66 to keep length to 62 +my $dbname3 = generate_ascii_string(130, 192); +my $dbname4 = generate_ascii_string(193, 255); + +my $node = get_new_node('main'); +$node->init(extra => ['--locale=C', '--encoding=LATIN1']); +# prep pg_hba.conf and pg_ident.conf +$node->run_log([$ENV{PG_REGRESS}, '--config-auth', $node->data_dir, + '--create-role', "$dbname1,$dbname2,$dbname3,$dbname4"]); +$node->start; + +my $backupdir = $node->backup_dir; +my $discard = "$backupdir/discard.sql"; +my $plain = "$backupdir/plain.sql"; +my $dirfmt = "$backupdir/dirfmt"; + +foreach my $dbname ($dbname1, $dbname2, $dbname3, $dbname4, 'CamelCase') +{ + $node->run_log(['createdb', $dbname]); + $node->run_log(['createuser', '-s', $dbname]); +} + + +# For these tests, pg_dumpall -r is used because it produces a short +# dump. +$node->command_ok(['pg_dumpall', '-r', '-f', $discard, '--dbname', + $node->connstr($dbname1), '-U', $dbname4], + 'pg_dumpall with long ASCII name 1'); +$node->command_ok(['pg_dumpall', '-r', '-f', $discard, '--dbname', + $node->connstr($dbname2), '-U', $dbname3], + 'pg_dumpall with long ASCII name 2'); +$node->command_ok(['pg_dumpall', '-r', '-f', $discard, '--dbname', + $node->connstr($dbname3), '-U', $dbname2], + 'pg_dumpall with long ASCII name 3'); +$node->command_ok(['pg_dumpall', '-r', '-f', $discard, '--dbname', + $node->connstr($dbname4), '-U', $dbname1], + 'pg_dumpall with long ASCII name 4'); +$node->command_ok(['pg_dumpall', '-r', '-l', 'dbname=template1'], + 'pg_dumpall -l accepts connection string'); + +$node->run_log(['createdb', "foo\n\rbar"]); +# not sufficient to use -r here +$node->command_fails(['pg_dumpall', '-f', $discard], + 'pg_dumpall with \n\r in database name'); +$node->run_log(['dropdb', "foo\n\rbar"]); + + +# make a table, so the parallel worker has something to dump +$node->safe_psql($dbname1, 'CREATE TABLE t0()'); +# XXX no printed message when this fails, just SIGPIPE termination +$node->command_ok(['pg_dump', '-Fd', '-j2', '-f', $dirfmt, + '-U', $dbname1, $node->connstr($dbname1)], + 'parallel dump'); + +# recreate $dbname1 for restore test +$node->run_log(['dropdb', $dbname1]); +$node->run_log(['createdb', $dbname1]); + +$node->command_ok(['pg_restore', '-v', '-d', 'template1', '-j2', + '-U', $dbname1, $dirfmt], + 'parallel restore'); + +$node->run_log(['dropdb', $dbname1]); + +$node->command_ok(['pg_restore', '-C', '-v', '-d', 'template1', '-j2', + '-U', $dbname1, $dirfmt], + 'parallel restore with create'); + + +$node->command_ok(['pg_dumpall', '-f', $plain, '-U', $dbname1], + 'take full dump'); +system_log('cat', $plain); +my($stderr, $result); +my $bootstrap_super = 'boot'; +my $restore_super = qq{a'b\\c=d\\ne"f}; + + +# Restore full dump through psql using environment variables for +# dbname/user connection parameters + +my $envar_node = get_new_node('destination_envar'); +$envar_node->init(extra => ['-U', $bootstrap_super, + '--locale=C', '--encoding=LATIN1']); +$envar_node->run_log([$ENV{PG_REGRESS}, + '--config-auth', $envar_node->data_dir, + '--create-role', "$bootstrap_super,$restore_super"]); +$envar_node->start; + +# make superuser for restore +$envar_node->run_log(['createuser', '-U', $bootstrap_super, '-s', $restore_super]); + +{ + local $ENV{PGPORT} = $envar_node->port; + local $ENV{PGUSER} = $restore_super; + $result = run_log(['psql', '-X', '-f', $plain], '2>', \$stderr); +} +ok($result, 'restore full dump using environment variables for connection parameters'); +is($stderr, '', 'no dump errors'); + + +# Restore full dump through psql using command-line options for +# dbname/user connection parameters. "\connect dbname=" forgets +# user/port from command line. + +$restore_super =~ s/"//g if $TestLib::windows_os; # IPC::Run mishandles '"' on Windows +my $cmdline_node = get_new_node('destination_cmdline'); +$cmdline_node->init(extra => ['-U', $bootstrap_super, + '--locale=C', '--encoding=LATIN1']); +$cmdline_node->run_log([$ENV{PG_REGRESS}, + '--config-auth', $cmdline_node->data_dir, + '--create-role', "$bootstrap_super,$restore_super"]); +$cmdline_node->start; +$cmdline_node->run_log(['createuser', '-U', $bootstrap_super, '-s', $restore_super]); +{ + $result = run_log(['psql', '-p', $cmdline_node->port, '-U', $restore_super, '-X', '-f', $plain], '2>', \$stderr); +} +ok($result, 'restore full dump with command-line options for connection parameters'); +is($stderr, '', 'no dump errors'); diff --git a/src/bin/pg_rewind/RewindTest.pm b/src/bin/pg_rewind/RewindTest.pm index 135d8f0449..c7c3a8f45c 100644 --- a/src/bin/pg_rewind/RewindTest.pm +++ b/src/bin/pg_rewind/RewindTest.pm @@ -133,7 +133,7 @@ sub create_standby $node_standby = get_new_node('standby'); $node_master->backup('my_backup'); $node_standby->init_from_backup($node_master, 'my_backup'); - my $connstr_master = $node_master->connstr('postgres'); + my $connstr_master = $node_master->connstr(); $node_standby->append_conf( "recovery.conf", qq( diff --git a/src/bin/scripts/t/010_clusterdb.pl b/src/bin/scripts/t/010_clusterdb.pl index 0e677cacf1..f3de9c016c 100644 --- a/src/bin/scripts/t/010_clusterdb.pl +++ b/src/bin/scripts/t/010_clusterdb.pl @@ -3,7 +3,7 @@ use PostgresNode; use TestLib; -use Test::More tests => 13; +use Test::More tests => 14; program_help_ok('clusterdb'); program_version_ok('clusterdb'); @@ -28,3 +28,6 @@ [ 'clusterdb', '-t', 'test1' ], qr/statement: CLUSTER test1;/, 'cluster specific table'); + +$node->command_ok([qw(clusterdb --echo --verbose dbname=template1)], + 'clusterdb with connection string'); diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl index d92896f34f..42d6fb4eb4 100644 --- a/src/bin/scripts/t/090_reindexdb.pl +++ b/src/bin/scripts/t/090_reindexdb.pl @@ -3,7 +3,7 @@ use PostgresNode; use TestLib; -use Test::More tests => 20; +use Test::More tests => 23; program_help_ok('reindexdb'); program_version_ok('reindexdb'); @@ -42,3 +42,10 @@ [ 'reindexdb', '-v', '-t', 'test1', 'postgres' ], qr/statement: REINDEX \(VERBOSE\) TABLE test1;/, 'reindex with verbose output'); + +$node->command_ok([qw(reindexdb --echo --table=pg_am dbname=template1)], + 'reindexdb table with connection string'); +$node->command_ok([qw(reindexdb --echo dbname=template1)], + 'reindexdb database with connection string'); +$node->command_ok([qw(reindexdb --echo --system dbname=template1)], + 'reindexdb system with connection string'); diff --git a/src/bin/scripts/t/100_vacuumdb.pl b/src/bin/scripts/t/100_vacuumdb.pl index c183ccb6a1..07c6e9e7ce 100644 --- a/src/bin/scripts/t/100_vacuumdb.pl +++ b/src/bin/scripts/t/100_vacuumdb.pl @@ -3,7 +3,7 @@ use PostgresNode; use TestLib; -use Test::More tests => 18; +use Test::More tests => 19; program_help_ok('vacuumdb'); program_version_ok('vacuumdb'); @@ -33,3 +33,5 @@ [ 'vacuumdb', '-Z', 'postgres' ], qr/statement: ANALYZE;/, 'vacuumdb -Z'); +$node->command_ok([qw(vacuumdb -Z --table=pg_am dbname=template1)], + 'vacuumdb with connection string'); diff --git a/src/bin/scripts/t/200_connstr.pl b/src/bin/scripts/t/200_connstr.pl new file mode 100644 index 0000000000..89945712d2 --- /dev/null +++ b/src/bin/scripts/t/200_connstr.pl @@ -0,0 +1,38 @@ +use strict; +use warnings; + +use PostgresNode; +use TestLib; +use Test::More tests => 3; + +# Tests to check connection string handling in utilities + +# In a SQL_ASCII database, pgwin32_message_to_UTF16() needs to +# interpret everything as UTF8. We're going to use byte sequences +# that aren't valid UTF-8 strings, so that would fail. Use LATIN1, +# which accepts any byte and has a conversion from each byte to UTF-8. +$ENV{LC_ALL} = 'C'; +$ENV{PGCLIENTENCODING} = 'LATIN1'; + +# Create database names covering the range of LATIN1 characters and +# run the utilities' --all options over them. +my $dbname1 = generate_ascii_string(1, 63); # contains '=' +my $dbname2 = generate_ascii_string(67, 129); # skip 64-66 to keep length to 62 +my $dbname3 = generate_ascii_string(130, 192); +my $dbname4 = generate_ascii_string(193, 255); + +my $node = get_new_node('main'); +$node->init(extra => ['--locale=C', '--encoding=LATIN1']); +$node->start; + +foreach my $dbname ($dbname1, $dbname2, $dbname3, $dbname4, 'CamelCase') +{ + $node->run_log(['createdb', $dbname]); +} + +$node->command_ok([qw(vacuumdb --all --echo --analyze-only)], + 'vacuumdb --all with unusual database names'); +$node->command_ok([qw(reindexdb --all --echo)], + 'reindexdb --all with unusual database names'); +$node->command_ok([qw(clusterdb --all --echo --verbose)], + 'clusterdb --all with unusual database names'); diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index fede1e601b..afbdb6332b 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -243,7 +243,13 @@ sub connstr { return "port=$pgport host=$pghost"; } - return "port=$pgport host=$pghost dbname=$dbname"; + + # Escape properly the database string before using it, only + # single quotes and backslashes need to be treated this way. + $dbname =~ s#\\#\\\\#g; + $dbname =~ s#\'#\\\'#g; + + return "port=$pgport host=$pghost dbname='$dbname'"; } =pod @@ -396,7 +402,8 @@ sub init mkdir $self->backup_dir; mkdir $self->archive_dir; - TestLib::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N'); + TestLib::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N', + @{ $params{extra} }); TestLib::system_or_bail($ENV{PG_REGRESS}, '--config-auth', $pgdata); open my $conf, ">>$pgdata/postgresql.conf"; @@ -1300,6 +1307,24 @@ sub issues_sql_like =pod +=item $node->run_log(...) + +Runs a shell command like TestLib::run_log, but with PGPORT set so +that the command will default to connecting to this PostgresNode. + +=cut + +sub run_log +{ + my $self = shift; + + local $ENV{PGPORT} = $self->port; + + TestLib::run_log(@_); +} + +=pod + =back =cut diff --git a/src/test/perl/TestLib.pm b/src/test/perl/TestLib.pm index 51b533e08c..31e7acd4da 100644 --- a/src/test/perl/TestLib.pm +++ b/src/test/perl/TestLib.pm @@ -20,6 +20,7 @@ use SimpleTee; use Test::More; our @EXPORT = qw( + generate_ascii_string slurp_dir slurp_file append_to_file @@ -166,6 +167,19 @@ sub run_log return IPC::Run::run(@_); } +# Generate a string made of the given range of ASCII characters +sub generate_ascii_string +{ + my ($from_char, $to_char) = @_; + my $res; + + for my $i ($from_char .. $to_char) + { + $res .= sprintf("%c", $i); + } + return $res; +} + sub slurp_dir { my ($dir) = @_; diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 14c87c91ab..1154d4c300 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -876,6 +876,29 @@ initialize_environment(void) load_resultmap(); } +pg_attribute_unused() +static const char * +fmtHba(const char *raw) +{ + static char *ret; + const char *rp; + char *wp; + + wp = ret = realloc(ret, 3 + strlen(raw) * 2); + + *wp++ = '"'; + for (rp = raw; *rp; rp++) + { + if (*rp == '"') + *wp++ = '"'; + *wp++ = *rp; + } + *wp++ = '"'; + *wp++ = '\0'; + + return ret; +} + #ifdef ENABLE_SSPI /* * Get account and domain/realm names for the current user. This is based on @@ -1037,11 +1060,11 @@ config_sspi_auth(const char *pgdata) * '#'. Windows forbids the double-quote character itself, so don't * bother escaping embedded double-quote characters. */ - CW(fprintf(ident, "regress \"%s@%s\" \"%s\"\n", - accountname, domainname, username) >= 0); + CW(fprintf(ident, "regress \"%s@%s\" %s\n", + accountname, domainname, fmtHba(username)) >= 0); for (sl = extraroles; sl; sl = sl->next) - CW(fprintf(ident, "regress \"%s@%s\" \"%s\"\n", - accountname, domainname, sl->str) >= 0); + CW(fprintf(ident, "regress \"%s@%s\" %s\n", + accountname, domainname, fmtHba(sl->str)) >= 0); CW(fclose(ident) == 0); } #endif @@ -2064,7 +2087,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc * before we add the specified one. */ free_stringlist(&dblist); - split_to_stringlist(optarg, ", ", &dblist); + split_to_stringlist(optarg, ",", &dblist); break; case 2: debug = true; @@ -2114,7 +2137,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc dlpath = pg_strdup(optarg); break; case 18: - split_to_stringlist(optarg, ", ", &extraroles); + split_to_stringlist(optarg, ",", &extraroles); break; case 19: add_stringlist_item(&temp_configs, optarg); From 96dd77d349424f270d129f8f40e75f762ddcca7d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 22 Sep 2016 11:34:44 -0400 Subject: [PATCH 203/871] Be sure to rewind the tuplestore read pointer in non-leader CTEScan nodes. ExecInitCteScan supposed that it didn't have to do anything to the extra tuplestore read pointer it gets from tuplestore_alloc_read_pointer. However, it needs this read pointer to be positioned at the start of the tuplestore, while tuplestore_alloc_read_pointer is actually defined as cloning the current position of read pointer 0. In normal situations that accidentally works because we initialize the whole plan tree at once, before anything gets read. But it fails in an EvalPlanQual recheck, as illustrated in bug #14328 from Dima Pavlov. To fix, just forcibly rewind the pointer after tuplestore_alloc_read_pointer. The cost of doing so is negligible unless the tuplestore is already in TSS_READFILE state, which wouldn't happen in normal cases. We could consider altering tuplestore's API to make that case cheaper, but that would make for a more invasive back-patch and it doesn't seem worth it. This has been broken probably for as long as we've had CTEs, so back-patch to all supported branches. Discussion: <32468.1474548308@sss.pgh.pa.us> --- src/backend/executor/nodeCtescan.c | 4 ++++ .../isolation/expected/eval-plan-qual.out | 21 +++++++++++++++++++ src/test/isolation/specs/eval-plan-qual.spec | 17 +++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/src/backend/executor/nodeCtescan.c b/src/backend/executor/nodeCtescan.c index 3c2f684a06..162650ad8a 100644 --- a/src/backend/executor/nodeCtescan.c +++ b/src/backend/executor/nodeCtescan.c @@ -224,9 +224,13 @@ ExecInitCteScan(CteScan *node, EState *estate, int eflags) { /* Not the leader */ Assert(IsA(scanstate->leader, CteScanState)); + /* Create my own read pointer, and ensure it is at start */ scanstate->readptr = tuplestore_alloc_read_pointer(scanstate->leader->cte_table, scanstate->eflags); + tuplestore_select_read_pointer(scanstate->leader->cte_table, + scanstate->readptr); + tuplestore_rescan(scanstate->leader->cte_table); } /* diff --git a/src/test/isolation/expected/eval-plan-qual.out b/src/test/isolation/expected/eval-plan-qual.out index 5898d94ff1..10c784a05f 100644 --- a/src/test/isolation/expected/eval-plan-qual.out +++ b/src/test/isolation/expected/eval-plan-qual.out @@ -163,3 +163,24 @@ ta_id ta_value tb_row 1 newTableAValue (1,tableBValue) step c2: COMMIT; + +starting permutation: wrtwcte readwcte c1 c2 +step wrtwcte: UPDATE table_a SET value = 'tableAValue2' WHERE id = 1; +step readwcte: + WITH + cte1 AS ( + SELECT id FROM table_b WHERE value = 'tableBValue' + ), + cte2 AS ( + SELECT * FROM table_a + WHERE id = (SELECT id FROM cte1) + FOR UPDATE + ) + SELECT * FROM cte2; + +step c1: COMMIT; +step c2: COMMIT; +step readwcte: <... completed> +id value + +1 tableAValue2 diff --git a/src/test/isolation/specs/eval-plan-qual.spec b/src/test/isolation/specs/eval-plan-qual.spec index de481a3cec..7ff6f6b8cc 100644 --- a/src/test/isolation/specs/eval-plan-qual.spec +++ b/src/test/isolation/specs/eval-plan-qual.spec @@ -103,11 +103,27 @@ step "readforss" { FROM table_a ta WHERE ta.id = 1 FOR UPDATE OF ta; } +step "wrtwcte" { UPDATE table_a SET value = 'tableAValue2' WHERE id = 1; } step "c2" { COMMIT; } session "s3" setup { BEGIN ISOLATION LEVEL READ COMMITTED; } step "read" { SELECT * FROM accounts ORDER BY accountid; } + +# this test exercises EvalPlanQual with a CTE, cf bug #14328 +step "readwcte" { + WITH + cte1 AS ( + SELECT id FROM table_b WHERE value = 'tableBValue' + ), + cte2 AS ( + SELECT * FROM table_a + WHERE id = (SELECT id FROM cte1) + FOR UPDATE + ) + SELECT * FROM cte2; +} + teardown { COMMIT; } permutation "wx1" "wx2" "c1" "c2" "read" @@ -118,3 +134,4 @@ permutation "writep2" "returningp1" "c1" "c2" permutation "wx2" "partiallock" "c2" "c1" "read" permutation "wx2" "lockwithvalues" "c2" "c1" "read" permutation "updateforss" "readforss" "c1" "c2" +permutation "wrtwcte" "readwcte" "c1" "c2" From 8023b5827fbada6815ce269db4f3373ac77ec7c3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 22 Sep 2016 14:30:33 -0400 Subject: [PATCH 204/871] Remove nearly-unused SizeOfIptrData macro. Past refactorings have removed all but one reference to SizeOfIptrData (and that one place was in a pretty noncritical spot). Since nobody's complained, it seems probable that there are no supported compilers that don't think sizeof(ItemPointerData) is 6. If there are, we're wasting MAXALIGN per heap tuple anyway, so it's rather silly to worry about whether we can shave space in places like WAL records. Pavan Deolasee Discussion: --- src/backend/executor/nodeTidscan.c | 2 +- src/include/storage/itemptr.h | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/backend/executor/nodeTidscan.c b/src/backend/executor/nodeTidscan.c index 2604103352..d54fe3665f 100644 --- a/src/backend/executor/nodeTidscan.c +++ b/src/backend/executor/nodeTidscan.c @@ -139,7 +139,7 @@ TidListCreate(TidScanState *tidstate) continue; itemarray = DatumGetArrayTypeP(arraydatum); deconstruct_array(itemarray, - TIDOID, SizeOfIptrData, false, 's', + TIDOID, sizeof(ItemPointerData), false, 's', &ipdatums, &ipnulls, &ndatums); if (numTids + ndatums > numAllocTids) { diff --git a/src/include/storage/itemptr.h b/src/include/storage/itemptr.h index 7ec7ed30c9..0deab7a859 100644 --- a/src/include/storage/itemptr.h +++ b/src/include/storage/itemptr.h @@ -30,9 +30,8 @@ * structure padding bytes. The struct is designed to be six bytes long * (it contains three int16 fields) but a few compilers will pad it to * eight bytes unless coerced. We apply appropriate persuasion where - * possible, and to cope with unpersuadable compilers, we try to use - * "SizeOfIptrData" rather than "sizeof(ItemPointerData)" when computing - * on-disk sizes. + * possible. If your compiler can't be made to play along, you'll waste + * lots of space. */ typedef struct ItemPointerData { @@ -46,9 +45,6 @@ pg_attribute_aligned(2) #endif ItemPointerData; -#define SizeOfIptrData \ - (offsetof(ItemPointerData, ip_posid) + sizeof(OffsetNumber)) - typedef ItemPointerData *ItemPointer; /* ---------------- From 1ff0042165cf835d1b6a2ab2c3b84c201eb6577b Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 22 Sep 2016 17:34:21 -0400 Subject: [PATCH 205/871] C comment: fix function header comment Fix for transformOnConflictClause(). Author: Tomonari Katsumata --- src/backend/parser/analyze.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index 870fae3f51..6901e08fd9 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -940,7 +940,7 @@ transformInsertRow(ParseState *pstate, List *exprlist, } /* - * transformSelectStmt - + * transformOnConflictClause - * transforms an OnConflictClause in an INSERT */ static OnConflictExpr * From 674e2de64d66b3a1e261bf07825be54c7cd2f3f6 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 23 Sep 2016 08:04:19 +0300 Subject: [PATCH 206/871] Fix typo in comment. Daniel Gustafsson --- src/interfaces/libpq/libpq-int.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index a94ead04ff..be6c3705b0 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -452,7 +452,7 @@ struct pg_conn #ifndef ENABLE_GSS gss_buffer_desc ginbuf; /* GSS input token */ #else - char *gsslib; /* What GSS librart to use ("gssapi" or + char *gsslib; /* What GSS library to use ("gssapi" or * "sspi") */ #endif CredHandle *sspicred; /* SSPI credentials handle */ From 3c2d5d6600de388729c7d613962d524352b7a67f Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 23 Sep 2016 14:21:59 +0300 Subject: [PATCH 207/871] Improve error message on MSVC if perl*.lib is not found. John Harvey, reviewed by Michael Paquier Discussion: --- src/tools/msvc/Mkvcbuild.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 93dfd24a83..07c91d1849 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -557,16 +557,17 @@ sub mkvcbuild } } $plperl->AddReference($postgres); + my $perl_path = $solution->{options}->{perl} . '\lib\CORE\perl*.lib'; my @perl_libs = grep { /perl\d+.lib$/ } - glob($solution->{options}->{perl} . '\lib\CORE\perl*.lib'); + glob($perl_path); if (@perl_libs == 1) { $plperl->AddLibrary($perl_libs[0]); } else { - die "could not identify perl library version"; + die "could not identify perl library version matching pattern $perl_path\n"; } # Add transform module dependent on plperl From 6fa51c79c7a645248a63205ce17af98ed404790d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 23 Sep 2016 12:00:00 -0400 Subject: [PATCH 208/871] pg_ctl: Add promote wait option to help output pointed out by Masahiko Sawada --- src/bin/pg_ctl/pg_ctl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index 2f0976a9cc..052b02e0ff 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -1941,9 +1941,9 @@ do_help(void) printf(_(" %s stop [-W] [-t SECS] [-D DATADIR] [-s] [-m SHUTDOWN-MODE]\n"), progname); printf(_(" %s restart [-w] [-t SECS] [-D DATADIR] [-s] [-m SHUTDOWN-MODE]\n" " [-o \"OPTIONS\"]\n"), progname); - printf(_(" %s reload [-D DATADIR] [-s]\n"), progname); - printf(_(" %s status [-D DATADIR]\n"), progname); - printf(_(" %s promote [-D DATADIR] [-s]\n"), progname); + printf(_(" %s reload [-D DATADIR] [-s]\n"), progname); + printf(_(" %s status [-D DATADIR]\n"), progname); + printf(_(" %s promote [-w] [-t SECS] [-D DATADIR] [-s]\n"), progname); printf(_(" %s kill SIGNALNAME PID\n"), progname); #ifdef WIN32 printf(_(" %s register [-N SERVICENAME] [-U USERNAME] [-P PASSWORD] [-D DATADIR]\n" From 49a91b88e6c4afb840745c78942dd99ce125a6d6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Sep 2016 09:54:11 -0400 Subject: [PATCH 209/871] Avoid using PostmasterRandom() for DSM control segment ID. Commits 470d886c3 et al intended to fix the problem that the postmaster selected the same "random" DSM control segment ID on every start. But using PostmasterRandom() for that destroys the intended property that the delay between random_start_time and random_stop_time will be unpredictable. (Said delay is probably already more predictable than we could wish, but that doesn't mean that reducing it by a couple orders of magnitude is OK.) Revert the previous patch and add a comment warning against misuse of PostmasterRandom. Fix the original problem by calling srandom() early in PostmasterMain, using a low-security seed that will later be overwritten by PostmasterRandom. Discussion: <20789.1474390434@sss.pgh.pa.us> --- src/backend/postmaster/postmaster.c | 17 ++++++++++++++++- src/backend/storage/ipc/dsm.c | 3 +-- src/include/postmaster/postmaster.h | 1 - 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 67ccdce764..40995580af 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -403,6 +403,7 @@ static void processCancelRequest(Port *port, void *pkt); static int initMasks(fd_set *rmask); static void report_fork_failure_to_client(Port *port, int errnum); static CAC_state canAcceptConnections(void); +static long PostmasterRandom(void); static void RandomSalt(char *salt, int len); static void signal_child(pid_t pid, int signal); static bool SignalSomeChildren(int signal, int targets); @@ -574,6 +575,16 @@ PostmasterMain(int argc, char *argv[]) */ umask(S_IRWXG | S_IRWXO); + /* + * Initialize random(3) so we don't get the same values in every run. + * + * Note: the seed is pretty predictable from externally-visible facts such + * as postmaster start time, so avoid using random() for security-critical + * random values during postmaster startup. At the time of first + * connection, PostmasterRandom will select a hopefully-more-random seed. + */ + srandom((unsigned int) (MyProcPid ^ MyStartTime)); + /* * By default, palloc() requests in the postmaster will be allocated in * the PostmasterContext, which is space that can be recycled by backends. @@ -5099,8 +5110,12 @@ RandomSalt(char *salt, int len) /* * PostmasterRandom + * + * Caution: use this only for values needed during connection-request + * processing. Otherwise, the intended property of having an unpredictable + * delay between random_start_time and random_stop_time will be broken. */ -long +static long PostmasterRandom(void) { /* diff --git a/src/backend/storage/ipc/dsm.c b/src/backend/storage/ipc/dsm.c index edafe4a3b9..d8066647a0 100644 --- a/src/backend/storage/ipc/dsm.c +++ b/src/backend/storage/ipc/dsm.c @@ -36,7 +36,6 @@ #include "lib/ilist.h" #include "miscadmin.h" -#include "postmaster/postmaster.h" #include "storage/dsm.h" #include "storage/ipc.h" #include "storage/lwlock.h" @@ -182,7 +181,7 @@ dsm_postmaster_startup(PGShmemHeader *shim) { Assert(dsm_control_address == NULL); Assert(dsm_control_mapped_size == 0); - dsm_control_handle = (dsm_handle) PostmasterRandom(); + dsm_control_handle = random(); if (dsm_control_handle == 0) continue; if (dsm_impl_op(DSM_OP_CREATE, dsm_control_handle, segsize, diff --git a/src/include/postmaster/postmaster.h b/src/include/postmaster/postmaster.h index ef06d5d04c..b2d7776f2a 100644 --- a/src/include/postmaster/postmaster.h +++ b/src/include/postmaster/postmaster.h @@ -48,7 +48,6 @@ extern const char *progname; extern void PostmasterMain(int argc, char *argv[]) pg_attribute_noreturn(); extern void ClosePostmasterPorts(bool am_syslogger); -extern long PostmasterRandom(void); extern int MaxLivePostmasterChildren(void); From 8e6b4ee21f486e6800aaa6955ff3d98e1a521b49 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Sep 2016 10:09:52 -0400 Subject: [PATCH 210/871] Don't trust CreateFileMapping() to clear the error code on success. We must test GetLastError() even when CreateFileMapping() returns a non-null handle. If that value were left over from some previous system call, we might be fooled into thinking the segment already existed. Experimentation on Windows 7 suggests that CreateFileMapping() clears the error code on success, but it is not documented to do so, so let's not rely on that happening in all Windows releases. Amit Kapila Discussion: <20811.1474390987@sss.pgh.pa.us> --- src/backend/storage/ipc/dsm_impl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/storage/ipc/dsm_impl.c b/src/backend/storage/ipc/dsm_impl.c index 7122c4abe0..dbac7bdac0 100644 --- a/src/backend/storage/ipc/dsm_impl.c +++ b/src/backend/storage/ipc/dsm_impl.c @@ -681,6 +681,9 @@ dsm_impl_windows(dsm_op op, dsm_handle handle, Size request_size, #endif size_low = (DWORD) request_size; + /* CreateFileMapping might not clear the error code on success */ + SetLastError(0); + hmap = CreateFileMapping(INVALID_HANDLE_VALUE, /* Use the pagefile */ NULL, /* Default security attrs */ PAGE_READWRITE, /* Memory is read/write */ From 959ea7fa76120449efa5d85ae351abd1d6e5480c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Sep 2016 10:44:50 -0400 Subject: [PATCH 211/871] Remove useless code. Apparent copy-and-pasteo in standby_desc_invalidations() had two entries for msg->id == SHAREDINVALRELMAP_ID. Aleksander Alekseev Discussion: <20160923090814.GB1238@e733> --- src/backend/access/rmgrdesc/standbydesc.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c index 13797a3d2f..8f19bb0a60 100644 --- a/src/backend/access/rmgrdesc/standbydesc.c +++ b/src/backend/access/rmgrdesc/standbydesc.c @@ -121,13 +121,11 @@ standby_desc_invalidations(StringInfo buf, else if (msg->id == SHAREDINVALSMGR_ID) appendStringInfoString(buf, " smgr"); /* not expected, but print something anyway */ - else if (msg->id == SHAREDINVALRELMAP_ID) - appendStringInfoString(buf, " relmap"); else if (msg->id == SHAREDINVALRELMAP_ID) appendStringInfo(buf, " relmap db %u", msg->rm.dbId); else if (msg->id == SHAREDINVALSNAPSHOT_ID) appendStringInfo(buf, " snapshot %u", msg->sn.relId); else - appendStringInfo(buf, " unknown id %d", msg->id); + appendStringInfo(buf, " unrecognized id %d", msg->id); } } From 12f6eadffd075e39570a0170f6791de0b96ddfde Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Sep 2016 13:49:26 -0400 Subject: [PATCH 212/871] Fix incorrect logic for excluding range constructor functions in pg_dump. Faulty AND/OR nesting in the WHERE clause of getFuncs' SQL query led to dumping range constructor functions if they are part of an extension and we're in binary-upgrade mode. Actually, we don't want to dump them separately even then, since CREATE TYPE AS RANGE will create the range's constructor functions regardless. Per report from Andrew Dunstan. It looks like this mistake was introduced by me, in commit b985d4877, in perhaps-overzealous refactoring to reduce code duplication. I'm suitably embarrassed. Report: <34854939-02d7-f591-5677-ce2994104599@dunslane.net> --- src/bin/pg_dump/pg_dump.c | 53 ++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index ba9c276593..51b8a1a622 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -4952,20 +4952,23 @@ getFuncs(Archive *fout, int *numFuncs) selectSourceSchema(fout, "pg_catalog"); /* - * Find all interesting functions. We include functions in pg_catalog, if - * they have an ACL different from what we set at initdb time (which is - * saved in pg_init_privs for us to perform this check). There may also - * be functions which are members of extensions which we must dump if we - * are in binary upgrade mode (we'll mark those functions as to-be-dumped - * when we check if the extension is to-be-dumped and we're in binary - * upgrade mode). + * Find all interesting functions. This is a bit complicated: * - * Also, in 9.2 and up, exclude functions that are internally dependent on - * something else, since presumably those will be created as a result of - * creating the something else. This currently only acts to suppress - * constructor functions for range types. Note that this is OK only - * because the constructors don't have any dependencies the range type - * doesn't have; otherwise we might not get creation ordering correct. + * 1. Always exclude aggregates; those are handled elsewhere. + * + * 2. Always exclude functions that are internally dependent on something + * else, since presumably those will be created as a result of creating + * the something else. This currently acts only to suppress constructor + * functions for range types (so we only need it in 9.2 and up). Note + * this is OK only because the constructors don't have any dependencies + * the range type doesn't have; otherwise we might not get creation + * ordering correct. + * + * 3. Otherwise, we normally exclude functions in pg_catalog. However, if + * they're members of extensions and we are in binary-upgrade mode then + * include them, since we want to dump extension members individually in + * that mode. Also, in 9.6 and up, include functions in pg_catalog if + * they have an ACL different from what's shown in pg_init_privs. */ if (fout->remoteVersion >= 90600) { @@ -4992,14 +4995,14 @@ getFuncs(Archive *fout, int *numFuncs) "(p.oid = pip.objoid " "AND pip.classoid = 'pg_proc'::regclass " "AND pip.objsubid = 0) " - "WHERE NOT proisagg " - "AND NOT EXISTS (SELECT 1 FROM pg_depend " + "WHERE NOT proisagg" + "\n AND NOT EXISTS (SELECT 1 FROM pg_depend " "WHERE classid = 'pg_proc'::regclass AND " - "objid = p.oid AND deptype = 'i') AND (" - "pronamespace != " + "objid = p.oid AND deptype = 'i')" + "\n AND (" + "\n pronamespace != " "(SELECT oid FROM pg_namespace " - "WHERE nspname = 'pg_catalog') OR " - "p.proacl IS DISTINCT FROM pip.initprivs", + "WHERE nspname = 'pg_catalog')", acl_subquery->data, racl_subquery->data, initacl_subquery->data, @@ -5012,6 +5015,8 @@ getFuncs(Archive *fout, int *numFuncs) "objid = p.oid AND " "refclassid = 'pg_extension'::regclass AND " "deptype = 'e')"); + appendPQExpBufferStr(query, + "\n OR p.proacl IS DISTINCT FROM pip.initprivs"); appendPQExpBufferChar(query, ')'); destroyPQExpBuffer(acl_subquery); @@ -5029,16 +5034,18 @@ getFuncs(Archive *fout, int *numFuncs) "pronamespace, " "(%s proowner) AS rolname " "FROM pg_proc p " - "WHERE NOT proisagg AND (" - "pronamespace != " - "(SELECT oid FROM pg_namespace " - "WHERE nspname = 'pg_catalog')", + "WHERE NOT proisagg", username_subquery); if (fout->remoteVersion >= 90200) appendPQExpBufferStr(query, "\n AND NOT EXISTS (SELECT 1 FROM pg_depend " "WHERE classid = 'pg_proc'::regclass AND " "objid = p.oid AND deptype = 'i')"); + appendPQExpBufferStr(query, + "\n AND (" + "\n pronamespace != " + "(SELECT oid FROM pg_namespace " + "WHERE nspname = 'pg_catalog')"); if (dopt->binary_upgrade && fout->remoteVersion >= 90100) appendPQExpBufferStr(query, "\n OR EXISTS(SELECT 1 FROM pg_depend WHERE " From 5a7bae0699657315b487896810a30bd7425f6a08 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Sep 2016 14:22:07 -0400 Subject: [PATCH 213/871] Doc: fix examples of # operators so they actually work. These worked as-is until around 7.0, but fail in newer versions because there are more operators named "#". Besides it's a bit inconsistent that only two of the examples on this page lack type names on their constants. Report: <20160923081530.1517.75670@wrigleys.postgresql.org> --- doc/src/sgml/func.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 47fcb30da0..3cc69bbffd 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -8365,12 +8365,12 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple # Point or box of intersection - '((1,-1),(-1,1))' # '((1,1),(-1,-1))' + box '((1,-1),(-1,1))' # box '((1,1),(-2,-2))' # Number of points in path or polygon - # '((1,0),(0,1),(-1,0))' + # path '((1,0),(0,1),(-1,0))' @-@ From c3a0818460a8f4a5e1a9109b035ac30e84b7e06f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Sep 2016 15:50:00 -0400 Subject: [PATCH 214/871] Install TAP test infrastructure so it's available for extension testing. When configured with --enable-tap-tests, "make install" will now install the Perl support files for TAP testing where PGXS will find them. This allows extensions to rely on $(prove_check) even when being built out-of-tree. Back-patch to 9.4 where we first started to support TAP testing, to reduce the number of cases extension makefiles need to consider. Craig Ringer Discussion: --- src/Makefile | 3 ++- src/test/Makefile | 2 +- src/test/perl/Makefile | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 src/test/perl/Makefile diff --git a/src/Makefile b/src/Makefile index b526be7985..977f80b469 100644 --- a/src/Makefile +++ b/src/Makefile @@ -26,7 +26,8 @@ SUBDIRS = \ bin \ pl \ makefiles \ - test/regress + test/regress \ + test/perl # There are too many interdependencies between the subdirectories, so # don't attempt parallel make here. diff --git a/src/test/Makefile b/src/test/Makefile index 7f7754f3ee..6b40cf50ed 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,7 +12,7 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = regress isolation modules recovery +SUBDIRS = perl regress isolation modules recovery # We don't build or execute examples/, locale/, or thread/ by default, # but we do want "make clean" etc to recurse into them. Likewise for ssl/, diff --git a/src/test/perl/Makefile b/src/test/perl/Makefile new file mode 100644 index 0000000000..8ab60fc326 --- /dev/null +++ b/src/test/perl/Makefile @@ -0,0 +1,33 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/test/perl +# +# Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/perl/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/test/perl +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +ifeq ($(enable_tap_tests),yes) + +installdirs: + $(MKDIR_P) '$(DESTDIR)$(pgxsdir)/$(subdir)' + +install: all installdirs + $(INSTALL_DATA) $(srcdir)/TestLib.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/TestLib.pm' + $(INSTALL_DATA) $(srcdir)/SimpleTee.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/SimpleTee.pm' + $(INSTALL_DATA) $(srcdir)/RecursiveCopy.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/RecursiveCopy.pm' + $(INSTALL_DATA) $(srcdir)/PostgresNode.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgresNode.pm' + +uninstall: + rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/TestLib.pm' + rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/SimpleTee.pm' + rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/RecursiveCopy.pm' + rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgresNode.pm' + +endif From 98c2d3332b30006ff71add99bc9d619c9457e71f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 24 Sep 2016 16:25:35 -0400 Subject: [PATCH 215/871] Do a final round of updates on the 9.6 release notes. Set release date, document a few recent commits, do one last pass of copy-editing. --- doc/src/sgml/catalogs.sgml | 9 +++- doc/src/sgml/release-9.6.sgml | 83 +++++++++++++++++++++++++++-------- 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 322d8d6dc7..29738b07cb 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7596,6 +7596,11 @@ application. + + By default, the pg_config view can be read + only by superusers. + + <structname>pg_config</> Columns @@ -7771,8 +7776,8 @@ - The pg_file_settings view can be read only by - superusers. + By default, the pg_file_settings view can be read + only by superusers.
diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index ddd280c85a..5c40910c72 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -6,8 +6,7 @@ Release Date - 2016-??-?? - Current as of 2016-08-27 (commit b9fe6cbc8) + 2016-09-29 @@ -56,7 +55,7 @@ Substantial performance improvements, especially in the area of - scalability on multi-CPU-socket servers + scalability on multi-CPU-socket servers @@ -269,7 +268,7 @@ This commit is also listed under libpq and psql Write (or its abbreviation ) explicitly to obtain the old - behavior. Scripts modified this way will still work with old + behavior. Scripts so modified will still work with old versions of psql. @@ -371,6 +370,7 @@ and many others in the same vein 2016-06-09 [c9ce4a1c6] Eliminate "parallel degree" terminology. 2016-06-16 [75be66464] Invent min_parallel_relation_size GUC to replace a hard- 2016-08-16 [f85b1a841] Disable parallel query by default. +2016-09-15 [72ce78162] Make min_parallel_relation_size's default value platform --> Parallel queries (Robert Haas, Amit Kapila, David Rowley, @@ -504,6 +504,7 @@ and many others in the same vein Improve sorting performance by using quicksort, not replacement @@ -693,7 +694,7 @@ and many others in the same vein (a,b) REFERENCES r (x,y), then a WHERE condition such as t.a = r.x AND t.b = r.y cannot select more than one r row per t row. - The planner formerly considered AND conditions + The planner formerly considered these AND conditions to be independent and would often drastically misestimate selectivity as a result. Now it compares the WHERE conditions to applicable foreign key constraints and produces @@ -731,7 +732,7 @@ and many others in the same vein containing only already-frozen tuples are identified in the table's visibility map, and can be skipped by vacuum even when doing transaction wraparound prevention. This should greatly reduce the - cost of maintaining large tables containing mostly-unchanged data. + cost of maintaining large tables containing mostly-unchanging data. @@ -872,7 +873,8 @@ and many others in the same vein from where it will be flushed to physical storage in due time. Many operating systems are not smart about managing this and allow large amounts of dirty data to accumulate before deciding to flush - it all at once, leading to long delays for new I/O requests. + it all at once, causing long delays for new I/O requests until the + flushing finishes. This change attempts to alleviate this problem by explicitly requesting data flushes after a configurable interval. @@ -1209,7 +1211,7 @@ and many others in the same vein 2016-04-08 [34c33a1f0] Add BSD authentication method. --> - Add a bsd authentication + Add a BSD authentication method to allow use of the BSD Authentication service for PostgreSQL client authentication (Marisa Emerson) @@ -1300,6 +1302,16 @@ and many others in the same vein + + Raise the maximum allowed value + of to 24 hours (Simon Riggs) + + + + + @@ -1346,9 +1358,9 @@ and many others in the same vein Making a distinction between these settings is no longer useful, - and is part of a planned future simplification of replication - setup. The old names are still accepted but are converted - internally. + and merging them is a step towards a planned future simplification + of replication setup. The old names are still accepted but are + converted to replica internally. @@ -1375,7 +1387,7 @@ and many others in the same vein --> Allow the server's SSL key file to have group read - access if owned by root (Christoph Berg) + access if it is owned by root (Christoph Berg) @@ -1616,7 +1628,7 @@ XXX this is pending backpatch, may need to remove Previously, such cases failed if the same target column was mentioned more than once, e.g., INSERT INTO tab (x[1], - x[2]) VALUES .... + x[2]) VALUES (...). @@ -1797,9 +1809,9 @@ XXX this is pending backpatch, may need to remove 2016-03-23 [473b93287] Support CREATE ACCESS METHOD --> - Introduce CREATE ACCESS METHOD to allow extensions - to create index access methods (Alexander Korotkov, Petr - Jelínek) + Introduce CREATE + ACCESS METHOD to allow extensions to create index access + methods (Alexander Korotkov, Petr Jelínek) @@ -1912,7 +1924,7 @@ XXX this is pending backpatch, may need to remove Formerly, many security-sensitive functions contained hard-wired checks that would throw an error if they were called by a - non-superuser role. This forced the use of superuser roles for + non-superuser. This forced the use of superuser roles for some relatively pedestrian tasks. The hard-wired error checks are now gone in favor of making initdb revoke the default public EXECUTE privilege on these functions. @@ -1931,6 +1943,11 @@ XXX this is pending backpatch, may need to remove that can be used to grant access to what were previously superuser-only functions (Stephen Frost) + + + Currently the only such role is pg_signal_backend, + but more are expected to be added in future. + @@ -2211,7 +2228,7 @@ XXX this is pending backpatch, may need to remove Allow ts_stat() and tsvector_update_trigger() to operate on values that are of types binary-compatible with the - expected argument type, not only that argument type; for example + expected argument type, not just exactly that type; for example allow citext where text is expected (Teodor Sigaev) @@ -2623,10 +2640,26 @@ This commit is also listed under psql and PL/pgSQL + + Allow pg_dump to dump non-extension-owned objects + that are within an extension-owned schema + (Martín Marqués) + + + + Previously such objects were ignored because they were mistakenly + assumed to belong to the extension owning their schema. + + + + + - In pg_dump, include the table name in object + In pg_dump output, include the table name in object tags for object types that are only uniquely named per-table (for example, triggers) (Peter Eisentraut) @@ -2912,6 +2945,7 @@ This commit is also listed under libpq and PL/pgSQL Allow changing the selection probabilities (weights) for scripts @@ -3011,6 +3045,7 @@ This commit is also listed under libpq and PL/pgSQL Speed up initdb by using just one @@ -3420,6 +3455,16 @@ This commit is also listed under libpq and PL/pgSQL + + + + Support OpenSSL 1.1.0 (Andreas Karlsson, Heikki Linnakangas) + + + From da6c4f6ca88df346573bdada2aa2544510bf167e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 25 Sep 2016 15:40:57 -0400 Subject: [PATCH 216/871] Refer to OS X as "macOS", except for the port name which is still "darwin". We weren't terribly consistent about whether to call Apple's OS "OS X" or "Mac OS X", and the former is probably confusing to people who aren't Apple users. Now that Apple has rebranded it "macOS", follow their lead to establish a consistent naming pattern. Also, avoid the use of the ancient project name "Darwin", except as the port code name which does not seem desirable to change. (In short, this patch touches documentation and comments, but no actual code.) I didn't touch contrib/start-scripts/osx/, either. I suspect those are obsolete and due for a rewrite, anyway. I dithered about whether to apply this edit to old release notes, but those were responsible for quite a lot of the inconsistencies, so I ended up changing them too. Anyway, Apple's being ahistorical about this, so why shouldn't we be? --- config/c-library.m4 | 4 ++-- configure | 8 ++++---- configure.in | 12 ++++++------ doc/src/sgml/client-auth.sgml | 2 +- doc/src/sgml/dfunc.sgml | 4 ++-- doc/src/sgml/docguide.sgml | 2 +- doc/src/sgml/install-windows.sgml | 2 +- doc/src/sgml/installation.sgml | 6 +++--- doc/src/sgml/monitoring.sgml | 2 +- doc/src/sgml/release-7.4.sgml | 14 +++++++------- doc/src/sgml/release-8.0.sgml | 10 +++++----- doc/src/sgml/release-8.1.sgml | 10 +++++----- doc/src/sgml/release-8.2.sgml | 4 ++-- doc/src/sgml/release-8.3.sgml | 4 ++-- doc/src/sgml/release-8.4.sgml | 6 +++--- doc/src/sgml/release-9.0.sgml | 8 ++++---- doc/src/sgml/release-9.1.sgml | 8 ++++---- doc/src/sgml/release-9.2.sgml | 6 +++--- doc/src/sgml/release-9.3.sgml | 10 +++++----- doc/src/sgml/release-9.4.sgml | 6 +++--- doc/src/sgml/release-9.5.sgml | 2 +- doc/src/sgml/release-old.sgml | 2 +- doc/src/sgml/runtime.sgml | 18 +++++++++--------- doc/src/sgml/uuid-ossp.sgml | 2 +- doc/src/sgml/wal.sgml | 2 +- src/Makefile.shlib | 2 +- src/backend/Makefile | 4 ++-- src/backend/port/dynloader/darwin.c | 2 +- src/backend/postmaster/postmaster.c | 2 +- src/backend/utils/adt/varlena.c | 4 ++-- src/backend/utils/misc/ps_status.c | 4 ++-- src/backend/utils/probes.d | 2 +- src/bin/psql/input.c | 2 +- src/include/port/darwin.h | 2 +- src/interfaces/libpq/fe-connect.c | 2 +- src/port/README | 2 +- src/port/chklocale.c | 2 +- src/template/darwin | 11 +++++++---- src/test/regress/pg_regress.c | 4 ++-- src/tools/find_typedef | 2 +- 40 files changed, 102 insertions(+), 99 deletions(-) diff --git a/config/c-library.m4 b/config/c-library.m4 index 50d068d3fb..d330b0cf95 100644 --- a/config/c-library.m4 +++ b/config/c-library.m4 @@ -292,8 +292,8 @@ AC_MSG_RESULT([$pgac_cv_snprintf_size_t_support]) # PGAC_TYPE_LOCALE_T # ------------------ -# Check for the locale_t type and find the right header file. Mac OS -# X needs xlocale.h; standard is locale.h, but glibc also has an +# Check for the locale_t type and find the right header file. macOS +# needs xlocale.h; standard is locale.h, but glibc also has an # xlocale.h file that we should not use. # AC_DEFUN([PGAC_TYPE_LOCALE_T], diff --git a/configure b/configure index 5fc8c442a2..55c771a11e 100755 --- a/configure +++ b/configure @@ -7658,9 +7658,9 @@ $as_echo "${python_libspec} ${python_additional_libs}" >&6; } if test "$python_enable_shared" != 1; then if test "$PORTNAME" = darwin; then - # OS X does supply a .dylib even though Py_ENABLE_SHARED does + # macOS does supply a .dylib even though Py_ENABLE_SHARED does # not get set. The file detection logic below doesn't succeed - # on older OS X versions, so make it explicit. + # on older macOS versions, so make it explicit. python_enable_shared=1 elif test "$PORTNAME" = win32; then # Windows also needs an explicit override. @@ -10120,7 +10120,7 @@ else fi elif test "$with_uuid" = e2fs ; then - # On OS X, the UUID functions are in libc + # On macOS, the UUID functions are in libc ac_fn_c_check_func "$LINENO" "uuid_generate" "ac_cv_func_uuid_generate" if test "x$ac_cv_func_uuid_generate" = xyes; then : UUID_LIBS="" @@ -12672,7 +12672,7 @@ cat >>confdefs.h <<_ACEOF #define HAVE_DECL_STRLCPY $ac_have_decl _ACEOF -# This is probably only present on Darwin, but may as well check always +# This is probably only present on macOS, but may as well check always ac_fn_c_check_decl "$LINENO" "F_FULLFSYNC" "ac_cv_have_decl_F_FULLFSYNC" "#include " if test "x$ac_cv_have_decl_F_FULLFSYNC" = xyes; then : diff --git a/configure.in b/configure.in index 96d865de9f..9850d993ff 100644 --- a/configure.in +++ b/configure.in @@ -943,9 +943,9 @@ if test "$with_python" = yes; then if test "$python_enable_shared" != 1; then if test "$PORTNAME" = darwin; then - # OS X does supply a .dylib even though Py_ENABLE_SHARED does + # macOS does supply a .dylib even though Py_ENABLE_SHARED does # not get set. The file detection logic below doesn't succeed - # on older OS X versions, so make it explicit. + # on older macOS versions, so make it explicit. python_enable_shared=1 elif test "$PORTNAME" = win32; then # Windows also needs an explicit override. @@ -1182,7 +1182,7 @@ if test "$with_uuid" = bsd ; then [UUID_LIBS=""], [AC_MSG_ERROR([BSD UUID functions are not present])]) elif test "$with_uuid" = e2fs ; then - # On OS X, the UUID functions are in libc + # On macOS, the UUID functions are in libc AC_CHECK_FUNC(uuid_generate, [UUID_LIBS=""], [AC_CHECK_LIB(uuid, uuid_generate, @@ -1425,8 +1425,8 @@ esac if test "$PORTNAME" != "win32"; then AC_SYS_LARGEFILE dnl Autoconf 2.69's AC_SYS_LARGEFILE believes it's a good idea to #define - dnl _DARWIN_USE_64_BIT_INODE, but it isn't: on OS X 10.5 that activates a - dnl bug that causes readdir() to sometimes return EINVAL. On later OS X + dnl _DARWIN_USE_64_BIT_INODE, but it isn't: on macOS 10.5 that activates a + dnl bug that causes readdir() to sometimes return EINVAL. On later macOS dnl versions where the feature actually works, it's on by default anyway. AH_VERBATIM([_DARWIN_USE_64_BIT_INODE],[]) fi @@ -1479,7 +1479,7 @@ fi AC_CHECK_DECLS(fdatasync, [], [], [#include ]) AC_CHECK_DECLS([strlcat, strlcpy]) -# This is probably only present on Darwin, but may as well check always +# This is probably only present on macOS, but may as well check always AC_CHECK_DECLS(F_FULLFSYNC, [], [], [#include ]) HAVE_IPV6=no diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index ca262d9452..a0d97ffbac 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -1306,7 +1306,7 @@ omicron bryanh guest1 socket parameter, or similar mechanisms. Currently that includes Linux, most flavors of BSD including - OS X, + macOS, and Solaris. diff --git a/doc/src/sgml/dfunc.sgml b/doc/src/sgml/dfunc.sgml index 5a368f6df0..ba2684cc3c 100644 --- a/doc/src/sgml/dfunc.sgml +++ b/doc/src/sgml/dfunc.sgml @@ -127,8 +127,8 @@ cc -shared -o foo.so foo.o - OS X - OS Xshared library + macOS + macOSshared library diff --git a/doc/src/sgml/docguide.sgml b/doc/src/sgml/docguide.sgml index 6f896b565f..48828aff37 100644 --- a/doc/src/sgml/docguide.sgml +++ b/doc/src/sgml/docguide.sgml @@ -275,7 +275,7 @@ apt-get install docbook docbook-dsssl docbook-xsl libxml2-utils openjade1.3 open - OS X + macOS If you use MacPorts, the following will get you set up: diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index 50116f315d..20fc47ae5f 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -51,7 +51,7 @@ MinGW-w64. These tools can also be used to cross-compile for 32 bit and 64 bit Windows targets on other hosts, such as Linux and - Darwin. + macOS. Cygwin is not recommended for running a production server, and it should only be used for running on older versions of Windows where diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index f6de18ed2d..5ee28fcf85 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -874,7 +874,7 @@ su - postgres Build with Bonjour support. This requires Bonjour support - in your operating system. Recommended on OS X. + in your operating system. Recommended on macOS. @@ -900,7 +900,7 @@ su - postgres @@ -2010,7 +2010,7 @@ kill `cat /usr/local/pgsql/data/postmaster.pid` PostgreSQL can be expected to work on these operating systems: Linux (all recent distributions), Windows (Win2000 SP4 and later), - FreeBSD, OpenBSD, NetBSD, OS X, AIX, HP/UX, Solaris, + FreeBSD, OpenBSD, NetBSD, macOS, AIX, HP/UX, Solaris, and UnixWare. Other Unix-like systems may also work but are not currently being tested. In most cases, all CPU architectures supported by a given operating system will work. Look in diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 077642878e..f400785721 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2739,7 +2739,7 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid, Currently, the DTrace utility is supported, which, at the time of this writing, is available - on Solaris, OS X, FreeBSD, NetBSD, and Oracle Linux. The + on Solaris, macOS, FreeBSD, NetBSD, and Oracle Linux. The SystemTap project for Linux provides a DTrace equivalent and can also be used. Supporting other dynamic tracing utilities is theoretically possible by changing the definitions for diff --git a/doc/src/sgml/release-7.4.sgml b/doc/src/sgml/release-7.4.sgml index 5a4c52d4c2..e42be5b89d 100644 --- a/doc/src/sgml/release-7.4.sgml +++ b/doc/src/sgml/release-7.4.sgml @@ -268,7 +268,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -2437,7 +2437,7 @@ aggregate plan Pretty-print UNION queries correctly Make psql handle \r\n newlines properly in COPY IN pg_dump handled ACLs with grant options incorrectly -Fix thread support for OS X and Solaris +Fix thread support for macOS and Solaris Updated JDBC driver (build 215) with various fixes ECPG fixes Translation updates (various contributors) @@ -2627,7 +2627,7 @@ memory error during COPY IN TABLE AS from tables without OIDs Fix problems with alter_table regression test during parallel testing -Fix problems with hitting open file limit, especially on OS X (Tom) +Fix problems with hitting open file limit, especially on macOS (Tom) Partial fix for Turkish-locale issues initdb will succeed now in Turkish locale, but there are still some inconveniences associated with the i/I problem. @@ -3256,7 +3256,7 @@ DROP SCHEMA information_schema CASCADE; - Enable PAM for Mac OS X (Aaron Hillegass) + Enable PAM for macOS (Aaron Hillegass) Make B-tree indexes fully WAL-safe (Tom) @@ -3539,9 +3539,9 @@ DROP SCHEMA information_schema CASCADE; - Add Mac OS X Rendezvous server support (Chris Campbell) + Add macOS Rendezvous server support (Chris Campbell) - This allows Mac OS X hosts to query the network for available + This allows macOS hosts to query the network for available PostgreSQL servers. @@ -4561,7 +4561,7 @@ DROP SCHEMA information_schema CASCADE; Fix locking code for s390x CPU (64-bit) (Tom) Allow OpenBSD to use local ident credentials (William Ahern) Make query plan trees read-only to executor (Tom) - Add Darwin startup scripts (David Wheeler) + Add macOS startup scripts (David Wheeler) Allow libpq to compile with Borland C++ compiler (Lester Godwin, Karl Waclawek) Use our own version of getopt_long() if needed (Peter) Convert administration scripts to C (Peter) diff --git a/doc/src/sgml/release-8.0.sgml b/doc/src/sgml/release-8.0.sgml index 299c34e0f0..becd5090cc 100644 --- a/doc/src/sgml/release-8.0.sgml +++ b/doc/src/sgml/release-8.0.sgml @@ -345,7 +345,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -1715,7 +1715,7 @@ While this could theoretically happen anywhere, no standard build of - Perl did things this way ... until Mac OS X 10.5. + Perl did things this way ... until macOS 10.5. @@ -2449,7 +2449,7 @@ Win32 to match the backend (Andrew) (Bruce) Fix pgxs -L library path -specification for Win32, Cygwin, OS X, AIX (Bruce) +specification for Win32, Cygwin, macOS, AIX (Bruce) Check that SID is enabled while checking for Win32 admin privileges (Magnus) @@ -5224,7 +5224,7 @@ typedefs (Michael) - Improvements to the Mac OS X startup scripts (Ray A.) + Improvements to the macOS startup scripts (Ray A.) @@ -5328,7 +5328,7 @@ typedefs (Michael) - Make libpq and ECPG build as proper shared libraries on OS X (Tom) + Make libpq and ECPG build as proper shared libraries on macOS (Tom) diff --git a/doc/src/sgml/release-8.1.sgml b/doc/src/sgml/release-8.1.sgml index 0cb5587e9b..05b07ade99 100644 --- a/doc/src/sgml/release-8.1.sgml +++ b/doc/src/sgml/release-8.1.sgml @@ -572,7 +572,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -2188,7 +2188,7 @@ While this could theoretically happen anywhere, no standard build of - Perl did things this way ... until Mac OS X 10.5. + Perl did things this way ... until macOS 10.5. @@ -2730,7 +2730,7 @@ - Fix for Darwin (OS X) compilation (Tom) + Fix for macOS (Darwin) compilation (Tom) @@ -3104,7 +3104,7 @@ Win32 to match the backend (Andrew) (Bruce) Fix pgxs -L library path -specification for Win32, Cygwin, OS X, AIX (Bruce) +specification for Win32, Cygwin, macOS, AIX (Bruce) Check that SID is enabled while checking for Win32 admin privileges (Magnus) @@ -5225,7 +5225,7 @@ SELECT CURRENT_TIMESTAMP AT TIME ZONE 'Europe/London'; Add support for fsync_writethrough on - Darwin (Chris Campbell) + macOS (Chris Campbell) diff --git a/doc/src/sgml/release-8.2.sgml b/doc/src/sgml/release-8.2.sgml index 7f6a74bac9..2d21728cf7 100644 --- a/doc/src/sgml/release-8.2.sgml +++ b/doc/src/sgml/release-8.2.sgml @@ -1487,7 +1487,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -3765,7 +3765,7 @@ While this could theoretically happen anywhere, no standard build of - Perl did things this way ... until Mac OS X 10.5. + Perl did things this way ... until macOS 10.5. diff --git a/doc/src/sgml/release-8.3.sgml b/doc/src/sgml/release-8.3.sgml index b56edb0102..b1b5d4875c 100644 --- a/doc/src/sgml/release-8.3.sgml +++ b/doc/src/sgml/release-8.3.sgml @@ -3075,7 +3075,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -8396,7 +8396,7 @@ current_date < 2017-11-17 Use SYSV semaphores rather than POSIX on Darwin - >= 6.0, i.e., OS X 10.2 and up (Chris Marcellino) + >= 6.0, i.e., macOS 10.2 and up (Chris Marcellino) diff --git a/doc/src/sgml/release-8.4.sgml b/doc/src/sgml/release-8.4.sgml index 8b16c9e9d3..0d0478855e 100644 --- a/doc/src/sgml/release-8.4.sgml +++ b/doc/src/sgml/release-8.4.sgml @@ -240,7 +240,7 @@ - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) @@ -5334,7 +5334,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -9764,7 +9764,7 @@ WITH w AS (SELECT * FROM foo) SELECT * FROM w, bar ... FOR UPDATE - Enable DTrace support on Mac OS X + Enable DTrace support on macOS Leopard and other non-Solaris platforms (Robert Lor) diff --git a/doc/src/sgml/release-9.0.sgml b/doc/src/sgml/release-9.0.sgml index 61dce9fd78..2238b53745 100644 --- a/doc/src/sgml/release-9.0.sgml +++ b/doc/src/sgml/release-9.0.sgml @@ -1541,7 +1541,7 @@ - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) @@ -2093,7 +2093,7 @@ - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) @@ -5895,7 +5895,7 @@ - Fix incorrect quoting of log file name in Mac OS X start script + Fix incorrect quoting of log file name in macOS start script (Sidar Lopez) @@ -10745,7 +10745,7 @@ if TG_OP = 'INSERT' and NEW.col1 = ... then - Bonjour support now requires OS X 10.3 or later. + Bonjour support now requires macOS 10.3 or later. The older API has been deprecated by Apple. diff --git a/doc/src/sgml/release-9.1.sgml b/doc/src/sgml/release-9.1.sgml index a66ca0d5b3..26b709056f 100644 --- a/doc/src/sgml/release-9.1.sgml +++ b/doc/src/sgml/release-9.1.sgml @@ -599,7 +599,7 @@ Branch: REL9_1_STABLE [354b3a3ac] 2016-06-19 14:01:17 -0400 This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). @@ -2937,7 +2937,7 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) @@ -3574,7 +3574,7 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) @@ -8443,7 +8443,7 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 - Fix incorrect quoting of log file name in Mac OS X start script + Fix incorrect quoting of log file name in macOS start script (Sidar Lopez) diff --git a/doc/src/sgml/release-9.2.sgml b/doc/src/sgml/release-9.2.sgml index c801f98c3f..0f6e3d127f 100644 --- a/doc/src/sgml/release-9.2.sgml +++ b/doc/src/sgml/release-9.2.sgml @@ -629,7 +629,7 @@ This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). @@ -3190,7 +3190,7 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) @@ -3899,7 +3899,7 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) diff --git a/doc/src/sgml/release-9.3.sgml b/doc/src/sgml/release-9.3.sgml index c75f1109e1..e321f4b31c 100644 --- a/doc/src/sgml/release-9.3.sgml +++ b/doc/src/sgml/release-9.3.sgml @@ -812,7 +812,7 @@ Branch: REL9_2_STABLE [37f30b251] 2016-04-18 13:19:52 -0400 This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). @@ -3021,7 +3021,7 @@ Branch: REL9_0_STABLE [4dddf8552] 2015-05-21 20:41:55 -0400 - Silence some build warnings on OS X (Tom Lane) + Silence some build warnings on macOS (Tom Lane) @@ -4092,7 +4092,7 @@ Branch: REL9_0_STABLE [2e4946169] 2015-01-07 22:46:20 -0500 - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) @@ -5743,7 +5743,7 @@ Branch: REL8_4_STABLE [ae41bb4be] 2014-05-30 18:18:32 -0400 - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) @@ -10710,7 +10710,7 @@ ALTER EXTENSION hstore UPDATE; Add instructions for setting - up the documentation tool chain on Mac OS X + up the documentation tool chain on macOS (Peter Eisentraut) diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index 443c772846..51896924c9 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -929,7 +929,7 @@ Branch: REL9_1_STABLE [de887cc8a] 2016-05-25 19:39:49 -0400 This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). @@ -5254,7 +5254,7 @@ Branch: REL9_3_STABLE [6347bdb31] 2015-04-05 13:01:55 -0400 - Silence some build warnings on OS X (Tom Lane) + Silence some build warnings on macOS (Tom Lane) @@ -5813,7 +5813,7 @@ Branch: REL9_0_STABLE [2e4946169] 2015-01-07 22:46:20 -0500 - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) diff --git a/doc/src/sgml/release-9.5.sgml b/doc/src/sgml/release-9.5.sgml index fa3537de10..c3f0f7051e 100644 --- a/doc/src/sgml/release-9.5.sgml +++ b/doc/src/sgml/release-9.5.sgml @@ -2023,7 +2023,7 @@ Branch: REL9_1_STABLE [e56acbe2a] 2016-02-10 19:30:12 -0500 This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). diff --git a/doc/src/sgml/release-old.sgml b/doc/src/sgml/release-old.sgml index ec8e43f6ea..cd9b3db35a 100644 --- a/doc/src/sgml/release-old.sgml +++ b/doc/src/sgml/release-old.sgml @@ -3299,7 +3299,7 @@ New BeOS port (David Reid, Cyril Velter) Add proofreader's changes to docs (Addison-Wesley, Bruce) New Alpha spinlock code (Adriaan Joubert, Compaq) UnixWare port overhaul (Peter E) -New Darwin/Mac OS X port (Peter Bierman, Bruce Hartzler) +New macOS (Darwin) port (Peter Bierman, Bruce Hartzler) New FreeBSD Alpha port (Alfred) Overhaul shared memory segments (Tom) Add IBM S/390 support (Neale Ferguson) diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index ef0139c365..88ec120841 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1006,12 +1006,12 @@ option SEMMAP=256 - OS X - OS XIPC configuration + macOS + macOSIPC configuration - The recommended method for configuring shared memory in OS X + The recommended method for configuring shared memory in macOS is to create a file named /etc/sysctl.conf, containing variable assignments such as: @@ -1021,13 +1021,13 @@ kern.sysv.shmmni=32 kern.sysv.shmseg=8 kern.sysv.shmall=1024 - Note that in some OS X versions, + Note that in some macOS versions, all five shared-memory parameters must be set in /etc/sysctl.conf, else the values will be ignored. - Beware that recent releases of OS X ignore attempts to set + Beware that recent releases of macOS ignore attempts to set SHMMAX to a value that isn't an exact multiple of 4096. @@ -1036,7 +1036,7 @@ kern.sysv.shmall=1024 - In older OS X versions, you will need to reboot to have changes in the + In older macOS versions, you will need to reboot to have changes in the shared memory parameters take effect. As of 10.5 it is possible to change all but SHMMNI on the fly, using sysctl. But it's still best to set up your preferred @@ -1045,7 +1045,7 @@ kern.sysv.shmall=1024 - The file /etc/sysctl.conf is only honored in OS X + The file /etc/sysctl.conf is only honored in macOS 10.3.9 and later. If you are running a previous 10.3.x release, you must edit the file /etc/rc and change the values in the following commands: @@ -1057,12 +1057,12 @@ sysctl -w kern.sysv.shmseg sysctl -w kern.sysv.shmall Note that - /etc/rc is usually overwritten by OS X system updates, + /etc/rc is usually overwritten by macOS system updates, so you should expect to have to redo these edits after each update. - In OS X 10.2 and earlier, instead edit these commands in the file + In macOS 10.2 and earlier, instead edit these commands in the file /System/Library/StartupItems/SystemTuning/SystemTuning. diff --git a/doc/src/sgml/uuid-ossp.sgml b/doc/src/sgml/uuid-ossp.sgml index e275febe4e..227d4a839c 100644 --- a/doc/src/sgml/uuid-ossp.sgml +++ b/doc/src/sgml/uuid-ossp.sgml @@ -169,7 +169,7 @@ SELECT uuid_generate_v3(uuid_ns_url(), 'http://www.postgresql.org'); platforms. uuid-ossp can now be built without the OSSP library on some platforms. On FreeBSD, NetBSD, and some other BSD-derived platforms, suitable UUID creation functions are included in the - core libc library. On Linux, OS X, and some other + core libc library. On Linux, macOS, and some other platforms, suitable functions are provided in the libuuid library, which originally came from the e2fsprogs project (though on modern Linux it is considered part diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index 9ae6547721..fe3b588c72 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -115,7 +115,7 @@ - On OS X, write caching can be prevented by + On macOS, write caching can be prevented by setting wal_sync_method to fsync_writethrough. diff --git a/src/Makefile.shlib b/src/Makefile.shlib index 924d21f443..de93f41639 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -323,7 +323,7 @@ endif endif # shlib_major # Where possible, restrict the symbols exported by the library to just the -# official list, so as to avoid unintentional ABI changes. On recent Darwin +# official list, so as to avoid unintentional ABI changes. On recent macOS # this also quiets multiply-defined-symbol warnings in programs that use # libpgport along with libpq. ifneq (,$(SHLIB_EXPORTS)) diff --git a/src/backend/Makefile b/src/backend/Makefile index 3b08defe2b..4946d37929 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -26,8 +26,8 @@ include $(srcdir)/common.mk # As of 1/2010: # The probes.o file is necessary for dtrace support on Solaris, and on recent # versions of systemtap. (Older systemtap releases just produce an empty -# file, but that's okay.) However, OS X's dtrace doesn't use it and doesn't -# even recognize the -G option. So, build probes.o except on Darwin. +# file, but that's okay.) However, macOS's dtrace doesn't use it and doesn't +# even recognize the -G option. So, build probes.o except on macOS. # This might need adjustment as other platforms add dtrace support. ifneq ($(PORTNAME), darwin) ifeq ($(enable_dtrace), yes) diff --git a/src/backend/port/dynloader/darwin.c b/src/backend/port/dynloader/darwin.c index a83c614f4f..7b6b48d14a 100644 --- a/src/backend/port/dynloader/darwin.c +++ b/src/backend/port/dynloader/darwin.c @@ -1,5 +1,5 @@ /* - * Dynamic loading support for Darwin + * Dynamic loading support for macOS (Darwin) * * If dlopen() is available (Darwin 10.3 and later), we just use it. * Otherwise we emulate it with the older, now deprecated, NSLinkModule API. diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 40995580af..0c0a609735 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -1273,7 +1273,7 @@ PostmasterMain(int argc, char *argv[]) #ifdef HAVE_PTHREAD_IS_THREADED_NP /* - * On Darwin, libintl replaces setlocale() with a version that calls + * On macOS, libintl replaces setlocale() with a version that calls * CFLocaleCopyCurrent() when its second argument is "" and every relevant * environment variable is unset or empty. CFLocaleCopyCurrent() makes * the process multithreaded. The postmaster calls sigprocmask() and diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 582d3e460b..260a5aac49 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -1844,8 +1844,8 @@ varstr_sortsupport(SortSupport ssup, Oid collid, bool bpchar) * Even apart from the risk of broken locales, it's possible that there * are platforms where the use of abbreviated keys should be disabled at * compile time. Having only 4 byte datums could make worst-case - * performance drastically more likely, for example. Moreover, Darwin's - * strxfrm() implementations is known to not effectively concentrate a + * performance drastically more likely, for example. Moreover, macOS's + * strxfrm() implementation is known to not effectively concentrate a * significant amount of entropy from the original string in earlier * transformed blobs. It's possible that other supported platforms are * similarly encumbered. So, if we ever get past disabling this diff --git a/src/backend/utils/misc/ps_status.c b/src/backend/utils/misc/ps_status.c index c50be8aab6..a889b170c8 100644 --- a/src/backend/utils/misc/ps_status.c +++ b/src/backend/utils/misc/ps_status.c @@ -223,8 +223,8 @@ save_ps_display_args(int argc, char **argv) #if defined(__darwin__) /* - * Darwin (and perhaps other NeXT-derived platforms?) has a static - * copy of the argv pointer, which we may fix like so: + * macOS (and perhaps other NeXT-derived platforms?) has a static copy + * of the argv pointer, which we may fix like so: */ *_NSGetArgv() = new_argv; #endif diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d index 976774e795..2f92dfa9ad 100644 --- a/src/backend/utils/probes.d +++ b/src/backend/utils/probes.d @@ -12,7 +12,7 @@ * Typedefs used in PostgreSQL. * * NOTE: Do not use system-provided typedefs (e.g. uintptr_t, uint32_t, etc) - * in probe definitions, as they cause compilation errors on Mac OS X 10.5. + * in probe definitions, as they cause compilation errors on macOS 10.5. */ #define LocalTransactionId unsigned int #define LWLockMode int diff --git a/src/bin/psql/input.c b/src/bin/psql/input.c index 2359b11dcd..a7d017a2d5 100644 --- a/src/bin/psql/input.c +++ b/src/bin/psql/input.c @@ -411,7 +411,7 @@ saveHistory(char *fname, int max_lines) /* * Suppressing the write attempt when HISTFILE is set to /dev/null may - * look like a negligible optimization, but it's necessary on e.g. Darwin, + * look like a negligible optimization, but it's necessary on e.g. macOS, * where write_history will fail because it tries to chmod the target * file. */ diff --git a/src/include/port/darwin.h b/src/include/port/darwin.h index 29c4b91d8c..15fb69d6db 100644 --- a/src/include/port/darwin.h +++ b/src/include/port/darwin.h @@ -2,7 +2,7 @@ #define __darwin__ 1 -#if HAVE_DECL_F_FULLFSYNC /* not present before OS X 10.3 */ +#if HAVE_DECL_F_FULLFSYNC /* not present before macOS 10.3 */ #define HAVE_FSYNC_WRITETHROUGH #endif diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 9668b52103..f3a9e5a83f 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -1252,7 +1252,7 @@ setKeepalivesIdle(PGconn *conn) } #else #ifdef TCP_KEEPALIVE - /* Darwin uses TCP_KEEPALIVE rather than TCP_KEEPIDLE */ + /* macOS uses TCP_KEEPALIVE rather than TCP_KEEPIDLE */ if (setsockopt(conn->sock, IPPROTO_TCP, TCP_KEEPALIVE, (char *) &idle, sizeof(idle)) < 0) { diff --git a/src/port/README b/src/port/README index 58fb32d9f9..4ae96da015 100644 --- a/src/port/README +++ b/src/port/README @@ -28,5 +28,5 @@ applications. from libpgport are linked first. This avoids having applications dependent on symbols that are _used_ by libpq, but not intended to be exported by libpq. libpq's libpgport usage changes over time, so such a -dependency is a problem. Win32, Linux, and Darwin use an export list to +dependency is a problem. Windows, Linux, and macOS use an export list to control the symbols exported by libpq. diff --git a/src/port/chklocale.c b/src/port/chklocale.c index 3c0ef6a253..915821a4e9 100644 --- a/src/port/chklocale.c +++ b/src/port/chklocale.c @@ -395,7 +395,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) #ifdef __darwin__ /* - * Current OS X has many locales that report an empty string for CODESET, + * Current macOS has many locales that report an empty string for CODESET, * but they all seem to actually use UTF-8. */ if (strlen(sys) == 0) diff --git a/src/template/darwin b/src/template/darwin index 542f706b0f..ea6d3b0b04 100644 --- a/src/template/darwin +++ b/src/template/darwin @@ -1,9 +1,12 @@ # src/template/darwin -# Select appropriate semaphore support. Darwin 6.0 (Mac OS X 10.2) and up -# support System V semaphores; before that we have to use POSIX semaphores, -# which are less good for our purposes because they eat a file descriptor -# per backend per max_connection slot. +# Note: Darwin is the original code name for macOS, also known as OS X. +# We still use "darwin" as the port name, partly because config.guess does. + +# Select appropriate semaphore support. Darwin 6.0 (macOS 10.2) and up +# support System V semaphores; before that we have to use named POSIX +# semaphores, which are less good for our purposes because they eat a +# file descriptor per backend per max_connection slot. case $host_os in darwin[015].*) USE_NAMED_POSIX_SEMAPHORES=1 diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 1154d4c300..b28cb0b1e1 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -738,9 +738,9 @@ initialize_environment(void) /* * Most platforms have adopted the POSIX locale as their * implementation-defined default locale. Exceptions include native - * Windows, Darwin with --enable-nls, and Cygwin with --enable-nls. + * Windows, macOS with --enable-nls, and Cygwin with --enable-nls. * (Use of --enable-nls matters because libintl replaces setlocale().) - * Also, PostgreSQL does not support Darwin with locale environment + * Also, PostgreSQL does not support macOS with locale environment * variables unset; see PostmasterMain(). */ #if defined(WIN32) || defined(__CYGWIN__) || defined(__darwin__) diff --git a/src/tools/find_typedef b/src/tools/find_typedef index fee0fb5152..24e9b76651 100755 --- a/src/tools/find_typedef +++ b/src/tools/find_typedef @@ -13,7 +13,7 @@ # find both .o files and executables. Therefore, ignore error messages about # unsuitable files being fed to objdump. # -# This is known to work on Linux and on some BSDen, including Mac OS X. +# This is known to work on Linux and on some BSDen, including macOS. # # Caution: on the platforms we use, this only prints typedefs that are used # to declare at least one variable or struct field. If you have say From 12788ae49e1933f463bc59a6efe46c4a01701b76 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 26 Sep 2016 10:56:02 +0300 Subject: [PATCH 217/871] Refactor script execution state machine in pgbench. The doCustom() function had grown into quite a mess. Rewrite it, in a more explicit state machine style, for readability. This also fixes one minor bug: if a script consisted entirely of meta commands, doCustom() never returned to the caller, so progress reports with the -P option were not printed. I don't want to backpatch this refactoring, and the bug is quite insignificant, so only commit this to master, and leave the bug unfixed in back-branches. Review and original bug report by Fabien Coelho. Discussion: --- src/bin/pgbench/pgbench.c | 1106 ++++++++++++++++++++++--------------- 1 file changed, 661 insertions(+), 445 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 8b24ad50e7..1fb4ae46d5 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -235,25 +235,95 @@ typedef struct StatsData } StatsData; /* - * Connection state + * Connection state machine states. + */ +typedef enum +{ + /* + * The client must first choose a script to execute. Once chosen, it can + * either be throttled (state CSTATE_START_THROTTLE under --rate) or start + * right away (state CSTATE_START_TX). + */ + CSTATE_CHOOSE_SCRIPT, + + /* + * In CSTATE_START_THROTTLE state, we calculate when to begin the next + * transaction, and advance to CSTATE_THROTTLE. CSTATE_THROTTLE state + * sleeps until that moment. (If throttling is not enabled, doCustom() + * falls directly through from CSTATE_START_THROTTLE to CSTATE_START_TX.) + */ + CSTATE_START_THROTTLE, + CSTATE_THROTTLE, + + /* + * CSTATE_START_TX performs start-of-transaction processing. Establishes + * a new connection for the transaction, in --connect mode, and records + * the transaction start time. + */ + CSTATE_START_TX, + + /* + * We loop through these states, to process each command in the script: + * + * CSTATE_START_COMMAND starts the execution of a command. On a SQL + * command, the command is sent to the server, and we move to + * CSTATE_WAIT_RESULT state. On a \sleep meta-command, the timer is set, + * and we enter the CSTATE_SLEEP state to wait for it to expire. Other + * meta-commands are executed immediately. + * + * CSTATE_WAIT_RESULT waits until we get a result set back from the server + * for the current command. + * + * CSTATE_SLEEP waits until the end of \sleep. + * + * CSTATE_END_COMMAND records the end-of-command timestamp, increments the + * command counter, and loops back to CSTATE_START_COMMAND state. + */ + CSTATE_START_COMMAND, + CSTATE_WAIT_RESULT, + CSTATE_SLEEP, + CSTATE_END_COMMAND, + + /* + * CSTATE_END_TX performs end-of-transaction processing. Calculates + * latency, and logs the transaction. In --connect mode, closes the + * current connection. Chooses the next script to execute and starts over + * in CSTATE_START_THROTTLE state, or enters CSTATE_FINISHED if we have no + * more work to do. + */ + CSTATE_END_TX, + + /* + * Final states. CSTATE_ABORTED means that the script execution was + * aborted because a command failed, CSTATE_FINISHED means success. + */ + CSTATE_ABORTED, + CSTATE_FINISHED +} ConnectionStateEnum; + +/* + * Connection state. */ typedef struct { PGconn *con; /* connection handle to DB */ int id; /* client No. */ - int state; /* state No. */ - bool listen; /* whether an async query has been sent */ - bool sleeping; /* whether the client is napping */ - bool throttling; /* whether nap is for throttling */ - bool is_throttled; /* whether transaction throttling is done */ + ConnectionStateEnum state; /* state machine's current state. */ + + int use_file; /* index in sql_script for this client */ + int command; /* command number in script */ + + /* client variables */ Variable *variables; /* array of variable definitions */ int nvariables; /* number of variables */ bool vars_sorted; /* are variables sorted by name? */ + + /* various times about current transaction */ int64 txn_scheduled; /* scheduled start time of transaction (usec) */ int64 sleep_until; /* scheduled start time of next cmd (usec) */ instr_time txn_begin; /* used for measuring schedule lag times */ instr_time stmt_begin; /* used for measuring statement latencies */ - int use_file; /* index in sql_scripts for this client */ + bool prepared[MAX_SCRIPTS]; /* whether client prepared the script */ /* per client collected stats */ @@ -1382,7 +1452,7 @@ evalFunc(TState *thread, CState *st, Assert(nargs == 1); fprintf(stderr, "debug(script=%d,command=%d): ", - st->use_file, st->state + 1); + st->use_file, st->command + 1); if (varg->type == PGBT_INT) fprintf(stderr, "int " INT64_FORMAT "\n", varg->u.ival); @@ -1733,15 +1803,12 @@ preparedStatementName(char *buffer, int file, int state) sprintf(buffer, "P%d_%d", file, state); } -static bool -clientDone(CState *st) +static void +commandFailed(CState *st, char *message) { - if (st->con != NULL) - { - PQfinish(st->con); - st->con = NULL; - } - return false; /* always false */ + fprintf(stderr, + "client %d aborted in command %d of script %d; %s\n", + st->id, st->command, st->use_file, message); } /* return a script number with a weighted choice. */ @@ -1763,425 +1830,595 @@ chooseScript(TState *thread) return i - 1; } -/* return false iff client should be disconnected */ +/* Send a SQL command, using the chosen querymode */ static bool -doCustom(TState *thread, CState *st, StatsData *agg) +sendCommand(CState *st, Command *command) { - PGresult *res; - Command **commands; - bool trans_needs_throttle = false; - instr_time now; + int r; - /* - * gettimeofday() isn't free, so we get the current timestamp lazily the - * first time it's needed, and reuse the same value throughout this - * function after that. This also ensures that e.g. the calculated latency - * reported in the log file and in the totals are the same. Zero means - * "not set yet". Reset "now" when we step to the next command with "goto - * top", though. - */ -top: - INSTR_TIME_SET_ZERO(now); + if (querymode == QUERY_SIMPLE) + { + char *sql; - commands = sql_script[st->use_file].commands; + sql = pg_strdup(command->argv[0]); + sql = assignVariables(st, sql); - /* - * Handle throttling once per transaction by sleeping. It is simpler to - * do this here rather than at the end, because so much complicated logic - * happens below when statements finish. - */ - if (throttle_delay && !st->is_throttled) + if (debug) + fprintf(stderr, "client %d sending %s\n", st->id, sql); + r = PQsendQuery(st->con, sql); + free(sql); + } + else if (querymode == QUERY_EXTENDED) { - /* - * Generate a delay such that the series of delays will approximate a - * Poisson distribution centered on the throttle_delay time. - * - * If transactions are too slow or a given wait is shorter than a - * transaction, the next transaction will start right away. - */ - int64 wait = getPoissonRand(thread, throttle_delay); + const char *sql = command->argv[0]; + const char *params[MAX_ARGS]; - thread->throttle_trigger += wait; - st->txn_scheduled = thread->throttle_trigger; + getQueryParams(st, command, params); - /* stop client if next transaction is beyond pgbench end of execution */ - if (duration > 0 && st->txn_scheduled > end_time) - return clientDone(st); + if (debug) + fprintf(stderr, "client %d sending %s\n", st->id, sql); + r = PQsendQueryParams(st->con, sql, command->argc - 1, + NULL, params, NULL, NULL, 0); + } + else if (querymode == QUERY_PREPARED) + { + char name[MAX_PREPARE_NAME]; + const char *params[MAX_ARGS]; - /* - * If this --latency-limit is used, and this slot is already late so - * that the transaction will miss the latency limit even if it - * completed immediately, we skip this time slot and iterate till the - * next slot that isn't late yet. - */ - if (latency_limit) + if (!st->prepared[st->use_file]) { - int64 now_us; + int j; + Command **commands = sql_script[st->use_file].commands; - if (INSTR_TIME_IS_ZERO(now)) - INSTR_TIME_SET_CURRENT(now); - now_us = INSTR_TIME_GET_MICROSEC(now); - while (thread->throttle_trigger < now_us - latency_limit) + for (j = 0; commands[j] != NULL; j++) { - processXactStats(thread, st, &now, true, agg); - /* next rendez-vous */ - wait = getPoissonRand(thread, throttle_delay); - thread->throttle_trigger += wait; - st->txn_scheduled = thread->throttle_trigger; + PGresult *res; + char name[MAX_PREPARE_NAME]; + + if (commands[j]->type != SQL_COMMAND) + continue; + preparedStatementName(name, st->use_file, j); + res = PQprepare(st->con, name, + commands[j]->argv[0], commands[j]->argc - 1, NULL); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + fprintf(stderr, "%s", PQerrorMessage(st->con)); + PQclear(res); } + st->prepared[st->use_file] = true; } - st->sleep_until = st->txn_scheduled; - st->sleeping = true; - st->throttling = true; - st->is_throttled = true; + getQueryParams(st, command, params); + preparedStatementName(name, st->use_file, st->command); + if (debug) - fprintf(stderr, "client %d throttling " INT64_FORMAT " us\n", - st->id, wait); + fprintf(stderr, "client %d sending %s\n", st->id, name); + r = PQsendQueryPrepared(st->con, name, command->argc - 1, + params, NULL, NULL, 0); } + else /* unknown sql mode */ + r = 0; - if (st->sleeping) - { /* are we sleeping? */ - if (INSTR_TIME_IS_ZERO(now)) - INSTR_TIME_SET_CURRENT(now); - if (INSTR_TIME_GET_MICROSEC(now) < st->sleep_until) - return true; /* Still sleeping, nothing to do here */ - /* Else done sleeping, go ahead with next command */ - st->sleeping = false; - st->throttling = false; + if (r == 0) + { + if (debug) + fprintf(stderr, "client %d could not send %s\n", + st->id, command->argv[0]); + st->ecnt++; + return false; } + else + return true; +} + +/* + * Parse the argument to a \sleep command, and return the requested amount + * of delay, in microseconds. Returns true on success, false on error. + */ +static bool +evaluateSleep(CState *st, int argc, char **argv, int *usecs) +{ + char *var; + int usec; - if (st->listen) - { /* are we receiver? */ - if (commands[st->state]->type == SQL_COMMAND) + if (*argv[1] == ':') + { + if ((var = getVariable(st, argv[1] + 1)) == NULL) { - if (debug) - fprintf(stderr, "client %d receiving\n", st->id); - if (!PQconsumeInput(st->con)) - { /* there's something wrong */ - fprintf(stderr, "client %d aborted in state %d; perhaps the backend died while processing\n", st->id, st->state); - return clientDone(st); - } - if (PQisBusy(st->con)) - return true; /* don't have the whole result yet */ + fprintf(stderr, "%s: undefined variable \"%s\"\n", + argv[0], argv[1]); + return false; } + usec = atoi(var); + } + else + usec = atoi(argv[1]); - /* - * command finished: accumulate per-command execution times in - * thread-local data structure, if per-command latencies are requested - */ - if (is_latencies) - { - if (INSTR_TIME_IS_ZERO(now)) - INSTR_TIME_SET_CURRENT(now); + if (argc > 2) + { + if (pg_strcasecmp(argv[2], "ms") == 0) + usec *= 1000; + else if (pg_strcasecmp(argv[2], "s") == 0) + usec *= 1000000; + } + else + usec *= 1000000; - /* XXX could use a mutex here, but we choose not to */ - addToSimpleStats(&commands[st->state]->stats, - INSTR_TIME_GET_DOUBLE(now) - - INSTR_TIME_GET_DOUBLE(st->stmt_begin)); - } + *usecs = usec; + return true; +} - /* transaction finished: calculate latency and log the transaction */ - if (commands[st->state + 1] == NULL) - { - if (progress || throttle_delay || latency_limit || - per_script_stats || use_log) - processXactStats(thread, st, &now, false, agg); - else - thread->stats.cnt++; - } +/* + * Advance the state machine of a connection, if possible. + */ +static void +doCustom(TState *thread, CState *st, StatsData *agg) +{ + PGresult *res; + Command *command; + instr_time now; + bool end_tx_processed = false; + int64 wait; - if (commands[st->state]->type == SQL_COMMAND) - { - /* - * Read and discard the query result; note this is not included in - * the statement latency numbers. - */ - res = PQgetResult(st->con); - switch (PQresultStatus(res)) - { - case PGRES_COMMAND_OK: - case PGRES_TUPLES_OK: - case PGRES_EMPTY_QUERY: - break; /* OK */ - default: - fprintf(stderr, "client %d aborted in state %d: %s", - st->id, st->state, PQerrorMessage(st->con)); - PQclear(res); - return clientDone(st); - } - PQclear(res); - discard_response(st); - } + /* + * gettimeofday() isn't free, so we get the current timestamp lazily the + * first time it's needed, and reuse the same value throughout this + * function after that. This also ensures that e.g. the calculated + * latency reported in the log file and in the totals are the same. Zero + * means "not set yet". Reset "now" when we execute shell commands or + * expressions, which might take a non-negligible amount of time, though. + */ + INSTR_TIME_SET_ZERO(now); - if (commands[st->state + 1] == NULL) + /* + * Loop in the state machine, until we have to wait for a result from the + * server (or have to sleep, for throttling or for \sleep). + * + * Note: In the switch-statement below, 'break' will loop back here, + * meaning "continue in the state machine". Return is used to return to + * the caller. + */ + for (;;) + { + switch (st->state) { - if (is_connect) - { - PQfinish(st->con); - st->con = NULL; - } + /* + * Select transaction to run. + */ + case CSTATE_CHOOSE_SCRIPT: - ++st->cnt; - if ((st->cnt >= nxacts && duration <= 0) || timer_exceeded) - return clientDone(st); /* exit success */ - } + st->use_file = chooseScript(thread); - /* increment state counter */ - st->state++; - if (commands[st->state] == NULL) - { - st->state = 0; - st->use_file = chooseScript(thread); - commands = sql_script[st->use_file].commands; - if (debug) - fprintf(stderr, "client %d executing script \"%s\"\n", st->id, - sql_script[st->use_file].desc); - st->is_throttled = false; - - /* - * No transaction is underway anymore, which means there is - * nothing to listen to right now. When throttling rate limits - * are active, a sleep will happen next, as the next transaction - * starts. And then in any case the next SQL command will set - * listen back to true. - */ - st->listen = false; - trans_needs_throttle = (throttle_delay > 0); - } - } + if (debug) + fprintf(stderr, "client %d executing script \"%s\"\n", st->id, + sql_script[st->use_file].desc); - if (st->con == NULL) - { - instr_time start, - end; + if (throttle_delay > 0) + st->state = CSTATE_START_THROTTLE; + else + st->state = CSTATE_START_TX; + break; - INSTR_TIME_SET_CURRENT(start); - if ((st->con = doConnect()) == NULL) - { - fprintf(stderr, "client %d aborted while establishing connection\n", - st->id); - return clientDone(st); - } - INSTR_TIME_SET_CURRENT(end); - INSTR_TIME_ACCUM_DIFF(thread->conn_time, end, start); - - /* Reset session-local state */ - st->listen = false; - st->sleeping = false; - st->throttling = false; - st->is_throttled = false; - memset(st->prepared, 0, sizeof(st->prepared)); - } + /* + * Handle throttling once per transaction by sleeping. + */ + case CSTATE_START_THROTTLE: - /* - * This ensures that a throttling delay is inserted before proceeding with - * sql commands, after the first transaction. The first transaction - * throttling is performed when first entering doCustom. - */ - if (trans_needs_throttle) - { - trans_needs_throttle = false; - goto top; - } + /* + * Generate a delay such that the series of delays will + * approximate a Poisson distribution centered on the + * throttle_delay time. + * + * If transactions are too slow or a given wait is shorter + * than a transaction, the next transaction will start right + * away. + */ + Assert(throttle_delay > 0); + wait = getPoissonRand(thread, throttle_delay); - /* Record transaction start time under logging, progress or throttling */ - if ((use_log || progress || throttle_delay || latency_limit || - per_script_stats) && st->state == 0) - { - INSTR_TIME_SET_CURRENT(st->txn_begin); + thread->throttle_trigger += wait; + st->txn_scheduled = thread->throttle_trigger; - /* - * When not throttling, this is also the transaction's scheduled start - * time. - */ - if (!throttle_delay) - st->txn_scheduled = INSTR_TIME_GET_MICROSEC(st->txn_begin); - } + /* + * stop client if next transaction is beyond pgbench end of + * execution + */ + if (duration > 0 && st->txn_scheduled > end_time) + { + st->state = CSTATE_FINISHED; + break; + } - /* Record statement start time if per-command latencies are requested */ - if (is_latencies) - INSTR_TIME_SET_CURRENT(st->stmt_begin); + /* + * If this --latency-limit is used, and this slot is already + * late so that the transaction will miss the latency limit + * even if it completed immediately, we skip this time slot + * and iterate till the next slot that isn't late yet. + */ + if (latency_limit) + { + int64 now_us; - if (commands[st->state]->type == SQL_COMMAND) - { - const Command *command = commands[st->state]; - int r; + if (INSTR_TIME_IS_ZERO(now)) + INSTR_TIME_SET_CURRENT(now); + now_us = INSTR_TIME_GET_MICROSEC(now); + while (thread->throttle_trigger < now_us - latency_limit) + { + processXactStats(thread, st, &now, true, agg); + /* next rendez-vous */ + wait = getPoissonRand(thread, throttle_delay); + thread->throttle_trigger += wait; + st->txn_scheduled = thread->throttle_trigger; + } + } - if (querymode == QUERY_SIMPLE) - { - char *sql; + st->state = CSTATE_THROTTLE; + if (debug) + fprintf(stderr, "client %d throttling " INT64_FORMAT " us\n", + st->id, wait); + break; - sql = pg_strdup(command->argv[0]); - sql = assignVariables(st, sql); + /* + * Wait until it's time to start next transaction. + */ + case CSTATE_THROTTLE: + if (INSTR_TIME_IS_ZERO(now)) + INSTR_TIME_SET_CURRENT(now); + if (INSTR_TIME_GET_MICROSEC(now) < st->txn_scheduled) + return; /* Still sleeping, nothing to do here */ + + /* Else done sleeping, start the transaction */ + st->state = CSTATE_START_TX; + break; - if (debug) - fprintf(stderr, "client %d sending %s\n", st->id, sql); - r = PQsendQuery(st->con, sql); - free(sql); - } - else if (querymode == QUERY_EXTENDED) - { - const char *sql = command->argv[0]; - const char *params[MAX_ARGS]; + /* Start new transaction */ + case CSTATE_START_TX: - getQueryParams(st, command, params); + /* + * Establish connection on first call, or if is_connect is + * true. + */ + if (st->con == NULL) + { + instr_time start; - if (debug) - fprintf(stderr, "client %d sending %s\n", st->id, sql); - r = PQsendQueryParams(st->con, sql, command->argc - 1, - NULL, params, NULL, NULL, 0); - } - else if (querymode == QUERY_PREPARED) - { - char name[MAX_PREPARE_NAME]; - const char *params[MAX_ARGS]; + if (INSTR_TIME_IS_ZERO(now)) + INSTR_TIME_SET_CURRENT(now); + start = now; + if ((st->con = doConnect()) == NULL) + { + fprintf(stderr, "client %d aborted while establishing connection\n", + st->id); + st->state = CSTATE_ABORTED; + break; + } + INSTR_TIME_SET_CURRENT(now); + INSTR_TIME_ACCUM_DIFF(thread->conn_time, now, start); - if (!st->prepared[st->use_file]) - { - int j; + /* Reset session-local state */ + memset(st->prepared, 0, sizeof(st->prepared)); + } - for (j = 0; commands[j] != NULL; j++) + /* + * Record transaction start time under logging, progress or + * throttling. + */ + if (use_log || progress || throttle_delay || latency_limit || + per_script_stats) { - PGresult *res; - char name[MAX_PREPARE_NAME]; + if (INSTR_TIME_IS_ZERO(now)) + INSTR_TIME_SET_CURRENT(now); + st->txn_begin = now; + + /* + * When not throttling, this is also the transaction's + * scheduled start time. + */ + if (!throttle_delay) + st->txn_scheduled = INSTR_TIME_GET_MICROSEC(now); + } - if (commands[j]->type != SQL_COMMAND) - continue; - preparedStatementName(name, st->use_file, j); - res = PQprepare(st->con, name, - commands[j]->argv[0], commands[j]->argc - 1, NULL); - if (PQresultStatus(res) != PGRES_COMMAND_OK) - fprintf(stderr, "%s", PQerrorMessage(st->con)); - PQclear(res); + /* Begin with the first command */ + st->command = 0; + st->state = CSTATE_START_COMMAND; + break; + + /* + * Send a command to server (or execute a meta-command) + */ + case CSTATE_START_COMMAND: + command = sql_script[st->use_file].commands[st->command]; + + /* + * If we reached the end of the script, move to end-of-xact + * processing. + */ + if (command == NULL) + { + st->state = CSTATE_END_TX; + break; } - st->prepared[st->use_file] = true; - } - getQueryParams(st, command, params); - preparedStatementName(name, st->use_file, st->state); + /* + * Record statement start time if per-command latencies are + * requested + */ + if (is_latencies) + { + if (INSTR_TIME_IS_ZERO(now)) + INSTR_TIME_SET_CURRENT(now); + st->stmt_begin = now; + } - if (debug) - fprintf(stderr, "client %d sending %s\n", st->id, name); - r = PQsendQueryPrepared(st->con, name, command->argc - 1, - params, NULL, NULL, 0); - } - else /* unknown sql mode */ - r = 0; + if (command->type == SQL_COMMAND) + { + if (!sendCommand(st, command)) + { + /* + * Failed. Stay in CSTATE_START_COMMAND state, to + * retry. ??? What the point or retrying? Should + * rather abort? + */ + return; + } + else + st->state = CSTATE_WAIT_RESULT; + } + else if (command->type == META_COMMAND) + { + int argc = command->argc, + i; + char **argv = command->argv; - if (r == 0) - { - if (debug) - fprintf(stderr, "client %d could not send %s\n", - st->id, command->argv[0]); - st->ecnt++; - } - else - st->listen = true; /* flags that should be listened */ - } - else if (commands[st->state]->type == META_COMMAND) - { - int argc = commands[st->state]->argc, - i; - char **argv = commands[st->state]->argv; + if (debug) + { + fprintf(stderr, "client %d executing \\%s", st->id, argv[0]); + for (i = 1; i < argc; i++) + fprintf(stderr, " %s", argv[i]); + fprintf(stderr, "\n"); + } - if (debug) - { - fprintf(stderr, "client %d executing \\%s", st->id, argv[0]); - for (i = 1; i < argc; i++) - fprintf(stderr, " %s", argv[i]); - fprintf(stderr, "\n"); - } + if (pg_strcasecmp(argv[0], "sleep") == 0) + { + /* + * A \sleep doesn't execute anything, we just get the + * delay from the argument, and enter the CSTATE_SLEEP + * state. (The per-command latency will be recorded + * in CSTATE_SLEEP state, not here, after the delay + * has elapsed.) + */ + int usec; + + if (!evaluateSleep(st, argc, argv, &usec)) + { + commandFailed(st, "execution of meta-command 'sleep' failed"); + st->state = CSTATE_ABORTED; + break; + } - if (pg_strcasecmp(argv[0], "set") == 0) - { - PgBenchExpr *expr = commands[st->state]->expr; - PgBenchValue result; + if (INSTR_TIME_IS_ZERO(now)) + INSTR_TIME_SET_CURRENT(now); + st->sleep_until = INSTR_TIME_GET_MICROSEC(now) + usec; + st->state = CSTATE_SLEEP; + break; + } + else + { + if (pg_strcasecmp(argv[0], "set") == 0) + { + PgBenchExpr *expr = command->expr; + PgBenchValue result; - if (!evaluateExpr(thread, st, expr, &result)) - { - st->ecnt++; - return true; - } + if (!evaluateExpr(thread, st, expr, &result)) + { + commandFailed(st, "evaluation of meta-command 'set' failed"); + st->state = CSTATE_ABORTED; + break; + } - if (!putVariableNumber(st, argv[0], argv[1], &result)) - { - st->ecnt++; - return true; - } + if (!putVariableNumber(st, argv[0], argv[1], &result)) + { + commandFailed(st, "assignment of meta-command 'set' failed"); + st->state = CSTATE_ABORTED; + break; + } + } + else if (pg_strcasecmp(argv[0], "setshell") == 0) + { + bool ret = runShellCommand(st, argv[1], argv + 2, argc - 2); - st->listen = true; - } - else if (pg_strcasecmp(argv[0], "sleep") == 0) - { - char *var; - int usec; - instr_time now; + if (timer_exceeded) /* timeout */ + { + st->state = CSTATE_FINISHED; + break; + } + else if (!ret) /* on error */ + { + commandFailed(st, "execution of meta-command 'setshell' failed"); + st->state = CSTATE_ABORTED; + break; + } + else + { + /* succeeded */ + } + } + else if (pg_strcasecmp(argv[0], "shell") == 0) + { + bool ret = runShellCommand(st, NULL, argv + 1, argc - 1); - if (*argv[1] == ':') - { - if ((var = getVariable(st, argv[1] + 1)) == NULL) + if (timer_exceeded) /* timeout */ + { + st->state = CSTATE_FINISHED; + break; + } + else if (!ret) /* on error */ + { + commandFailed(st, "execution of meta-command 'shell' failed"); + st->state = CSTATE_ABORTED; + break; + } + else + { + /* succeeded */ + } + } + + /* + * executing the expression or shell command might + * take a non-negligible amount of time, so reset + * 'now' + */ + INSTR_TIME_SET_ZERO(now); + + st->state = CSTATE_END_COMMAND; + } + } + break; + + /* + * Wait for the current SQL command to complete + */ + case CSTATE_WAIT_RESULT: + command = sql_script[st->use_file].commands[st->command]; + if (debug) + fprintf(stderr, "client %d receiving\n", st->id); + if (!PQconsumeInput(st->con)) + { /* there's something wrong */ + commandFailed(st, "perhaps the backend died while processing"); + st->state = CSTATE_ABORTED; + break; + } + if (PQisBusy(st->con)) + return; /* don't have the whole result yet */ + + /* + * Read and discard the query result; + */ + res = PQgetResult(st->con); + switch (PQresultStatus(res)) { - fprintf(stderr, "%s: undefined variable \"%s\"\n", - argv[0], argv[1]); - st->ecnt++; - return true; + case PGRES_COMMAND_OK: + case PGRES_TUPLES_OK: + case PGRES_EMPTY_QUERY: + /* OK */ + PQclear(res); + discard_response(st); + st->state = CSTATE_END_COMMAND; + break; + default: + commandFailed(st, PQerrorMessage(st->con)); + PQclear(res); + st->state = CSTATE_ABORTED; + break; } - usec = atoi(var); - } - else - usec = atoi(argv[1]); + break; - if (argc > 2) - { - if (pg_strcasecmp(argv[2], "ms") == 0) - usec *= 1000; - else if (pg_strcasecmp(argv[2], "s") == 0) - usec *= 1000000; - } - else - usec *= 1000000; + /* + * Wait until sleep is done. This state is entered after a + * \sleep metacommand. The behavior is similar to + * CSTATE_THROTTLE, but proceeds to CSTATE_START_COMMAND + * instead of CSTATE_START_TX. + */ + case CSTATE_SLEEP: + if (INSTR_TIME_IS_ZERO(now)) + INSTR_TIME_SET_CURRENT(now); + if (INSTR_TIME_GET_MICROSEC(now) < st->sleep_until) + return; /* Still sleeping, nothing to do here */ + /* Else done sleeping. */ + st->state = CSTATE_END_COMMAND; + break; - INSTR_TIME_SET_CURRENT(now); - st->sleep_until = INSTR_TIME_GET_MICROSEC(now) + usec; - st->sleeping = true; + /* + * End of command: record stats and proceed to next command. + */ + case CSTATE_END_COMMAND: - st->listen = true; - } - else if (pg_strcasecmp(argv[0], "setshell") == 0) - { - bool ret = runShellCommand(st, argv[1], argv + 2, argc - 2); + /* + * command completed: accumulate per-command execution times + * in thread-local data structure, if per-command latencies + * are requested. + */ + if (is_latencies) + { + if (INSTR_TIME_IS_ZERO(now)) + INSTR_TIME_SET_CURRENT(now); - if (timer_exceeded) /* timeout */ - return clientDone(st); - else if (!ret) /* on error */ - { - st->ecnt++; - return true; - } - else /* succeeded */ - st->listen = true; - } - else if (pg_strcasecmp(argv[0], "shell") == 0) - { - bool ret = runShellCommand(st, NULL, argv + 1, argc - 1); + /* XXX could use a mutex here, but we choose not to */ + command = sql_script[st->use_file].commands[st->command]; + addToSimpleStats(&command->stats, + INSTR_TIME_GET_DOUBLE(now) - + INSTR_TIME_GET_DOUBLE(st->stmt_begin)); + } - if (timer_exceeded) /* timeout */ - return clientDone(st); - else if (!ret) /* on error */ - { - st->ecnt++; - return true; - } - else /* succeeded */ - st->listen = true; - } + /* Go ahead with next command */ + st->command++; + st->state = CSTATE_START_COMMAND; + break; - /* after a meta command, immediately proceed with next command */ - goto top; - } + /* + * End of transaction. + */ + case CSTATE_END_TX: - return true; + /* + * transaction finished: calculate latency and log the + * transaction + */ + if (progress || throttle_delay || latency_limit || + per_script_stats || use_log) + processXactStats(thread, st, &now, false, agg); + else + thread->stats.cnt++; + + if (is_connect) + { + PQfinish(st->con); + st->con = NULL; + INSTR_TIME_SET_ZERO(now); + } + + ++st->cnt; + if ((st->cnt >= nxacts && duration <= 0) || timer_exceeded) + { + /* exit success */ + st->state = CSTATE_FINISHED; + break; + } + + /* + * No transaction is underway anymore. + */ + st->state = CSTATE_CHOOSE_SCRIPT; + + /* + * If we paced through all commands in the script in this + * loop, without returning to the caller even once, do it now. + * This gives the thread a chance to process other + * connections, and to do progress reporting. This can + * currently only happen if the script consists entirely of + * meta-commands. + */ + if (end_tx_processed) + return; + else + { + end_tx_processed = true; + break; + } + + /* + * Final states. Close the connection if it's still open. + */ + case CSTATE_ABORTED: + case CSTATE_FINISHED: + if (st->con != NULL) + { + PQfinish(st->con); + st->con = NULL; + } + return; + } + } } /* @@ -4183,29 +4420,10 @@ threadRun(void *arg) initStats(&aggs, INSTR_TIME_GET_DOUBLE(thread->start_time)); last = aggs; - /* send start up queries in async manner */ + /* initialize explicitely the state machines */ for (i = 0; i < nstate; i++) { - CState *st = &state[i]; - int prev_ecnt = st->ecnt; - Command **commands; - - st->use_file = chooseScript(thread); - commands = sql_script[st->use_file].commands; - if (debug) - fprintf(stderr, "client %d executing script \"%s\"\n", st->id, - sql_script[st->use_file].desc); - if (!doCustom(thread, st, &aggs)) - remains--; /* I've aborted */ - - if (st->ecnt > prev_ecnt && commands[st->state]->type == META_COMMAND) - { - fprintf(stderr, "client %d aborted in state %d; execution of meta-command failed\n", - i, st->state); - remains--; /* I've aborted */ - PQfinish(st->con); - st->con = NULL; - } + state[i].state = CSTATE_CHOOSE_SCRIPT; } while (remains > 0) @@ -4222,59 +4440,60 @@ threadRun(void *arg) for (i = 0; i < nstate; i++) { CState *st = &state[i]; - Command **commands = sql_script[st->use_file].commands; int sock; - if (st->con == NULL) + if (st->state == CSTATE_THROTTLE && timer_exceeded) { + /* interrupt client which has not started a transaction */ + st->state = CSTATE_FINISHED; + remains--; + PQfinish(st->con); + st->con = NULL; continue; } - else if (st->sleeping) + else if (st->state == CSTATE_SLEEP || st->state == CSTATE_THROTTLE) { - if (st->throttling && timer_exceeded) - { - /* interrupt client which has not started a transaction */ - remains--; - st->sleeping = false; - st->throttling = false; - PQfinish(st->con); - st->con = NULL; - continue; - } - else /* just a nap from the script */ - { - int this_usec; + /* a nap from the script, or under throttling */ + int this_usec; - if (min_usec == PG_INT64_MAX) - { - instr_time now; - - INSTR_TIME_SET_CURRENT(now); - now_usec = INSTR_TIME_GET_MICROSEC(now); - } + if (min_usec == PG_INT64_MAX) + { + instr_time now; - this_usec = st->txn_scheduled - now_usec; - if (min_usec > this_usec) - min_usec = this_usec; + INSTR_TIME_SET_CURRENT(now); + now_usec = INSTR_TIME_GET_MICROSEC(now); } + + this_usec = (st->state == CSTATE_SLEEP ? + st->sleep_until : st->txn_scheduled) - now_usec; + if (min_usec > this_usec) + min_usec = this_usec; } - else if (commands[st->state]->type == META_COMMAND) + else if (st->state == CSTATE_WAIT_RESULT) { - min_usec = 0; /* the connection is ready to run */ + /* + * waiting for result from server - nothing to do unless the + * socket is readable + */ + sock = PQsocket(st->con); + if (sock < 0) + { + fprintf(stderr, "invalid socket: %s", PQerrorMessage(st->con)); + goto done; + } + + FD_SET(sock, &input_mask); + + if (maxsock < sock) + maxsock = sock; break; } - - sock = PQsocket(st->con); - if (sock < 0) + else if (st->state != CSTATE_ABORTED && st->state != CSTATE_FINISHED) { - fprintf(stderr, "invalid socket: %s", PQerrorMessage(st->con)); - goto done; + /* the connection is ready to run */ + min_usec = 0; + break; } - - FD_SET(sock, &input_mask); - - if (maxsock < sock) - maxsock = sock; } /* also wake up to print the next progress report on time */ @@ -4324,14 +4543,13 @@ threadRun(void *arg) } } - /* ok, backend returns reply */ + /* ok, advance the state machine of each connection */ for (i = 0; i < nstate; i++) { CState *st = &state[i]; - Command **commands = sql_script[st->use_file].commands; - int prev_ecnt = st->ecnt; + bool ready; - if (st->con) + if (st->state == CSTATE_WAIT_RESULT && st->con) { int sock = PQsocket(st->con); @@ -4341,21 +4559,19 @@ threadRun(void *arg) PQerrorMessage(st->con)); goto done; } - if (FD_ISSET(sock, &input_mask) || - commands[st->state]->type == META_COMMAND) - { - if (!doCustom(thread, st, &aggs)) - remains--; /* I've aborted */ - } + + ready = FD_ISSET(sock, &input_mask); } + else if (st->state == CSTATE_FINISHED || st->state == CSTATE_ABORTED) + ready = false; + else + ready = true; - if (st->ecnt > prev_ecnt && commands[st->state]->type == META_COMMAND) + if (ready) { - fprintf(stderr, "client %d aborted in state %d; execution of meta-command failed\n", - i, st->state); - remains--; /* I've aborted */ - PQfinish(st->con); - st->con = NULL; + doCustom(thread, st, &aggs); + if (st->state == CSTATE_FINISHED || st->state == CSTATE_ABORTED) + remains--; } } From a4afb2b5c0b409bb175c20104b2ae9d47cf71be6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 26 Sep 2016 11:50:35 -0400 Subject: [PATCH 218/871] Document has_type_privilege(). Evidently an oversight in commit 729205571. Back-patch to 9.2 where privileges for types were introduced. Report: <20160922173517.8214.88959@wrigleys.postgresql.org> --- doc/src/sgml/func.sgml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 3cc69bbffd..67de029c6a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -15897,6 +15897,21 @@ SET search_path TO schema , schema, .. boolean does current user have privilege for tablespace + + has_type_privilege(user, + type, + privilege) + + boolean + does user have privilege for type + + + has_type_privilege(type, + privilege) + + boolean + does current user have privilege for type + pg_has_role(user, role, @@ -15955,6 +15970,9 @@ SET search_path TO schema , schema, .. has_tablespace_privilege + + has_type_privilege + pg_has_role @@ -16109,6 +16127,18 @@ SELECT has_function_privilege('joeuser', 'myfunc(int, text)', 'execute'); CREATE. + + has_type_privilege checks whether a user + can access a type in a particular way. + Its argument possibilities + are analogous to has_table_privilege. + When specifying a type by a text string rather than by OID, + the allowed input is the same as for the regtype data type + (see ). + The desired access privilege type must evaluate to + USAGE. + + pg_has_role checks whether a user can access a role in a particular way. From fdc9186f7ed1ead827509584f3b763f8dc332c43 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 26 Sep 2016 14:52:44 -0400 Subject: [PATCH 219/871] Replace the built-in GIN array opclasses with a single polymorphic opclass. We had thirty different GIN array opclasses sharing the same operators and support functions. That still didn't cover all the built-in types, nor did it cover arrays of extension-added types. What we want is a single polymorphic opclass for "anyarray". There were two missing features needed to make this possible: 1. We have to be able to declare the index storage type as ANYELEMENT when the opclass is declared to index ANYARRAY. This just takes a few more lines in index_create(). Although this currently seems of use only for GIN, there's no reason to make index_create() restrict it to that. 2. We have to be able to identify the proper GIN compare function for the index storage type. This patch proceeds by making the compare function optional in GIN opclass definitions, and specifying that the default btree comparison function for the index storage type will be looked up when the opclass omits it. Again, that seems pretty generically useful. Since the comparison function lookup is done in initGinState(), making use of the second feature adds an additional cache lookup to GIN index access setup. It seems unlikely that that would be very noticeable given the other costs involved, but maybe at some point we should consider making GinState data persist longer than it now does --- we could keep it in the index relcache entry, perhaps. Rather fortuitously, we don't seem to need to do anything to get this change to play nice with dump/reload or pg_upgrade scenarios: the new opclass definition is automatically selected to replace existing index definitions, and the on-disk data remains compatible. Also, if a user has created a custom opclass definition for a non-builtin type, this doesn't break that, since CREATE INDEX will prefer an exact match to opcintype over a match to ANYARRAY. However, if there's anyone out there with handwritten DDL that explicitly specifies _bool_ops or one of the other replaced opclass names, they'll need to adjust that. Tom Lane, reviewed by Enrique Meneses Discussion: <14436.1470940379@sss.pgh.pa.us> --- doc/src/sgml/gin.sgml | 349 +++------------------------ doc/src/sgml/indices.sgml | 5 +- doc/src/sgml/ref/create_opclass.sgml | 5 + doc/src/sgml/xindex.sgml | 2 +- src/backend/access/gin/ginutil.c | 32 ++- src/backend/access/gin/ginvalidate.c | 2 +- src/backend/catalog/index.c | 19 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_amop.h | 3 +- src/include/catalog/pg_amproc.h | 154 +----------- src/include/catalog/pg_opclass.h | 31 +-- 11 files changed, 101 insertions(+), 503 deletions(-) diff --git a/doc/src/sgml/gin.sgml b/doc/src/sgml/gin.sgml index 05d92eb975..7c2321ec3c 100644 --- a/doc/src/sgml/gin.sgml +++ b/doc/src/sgml/gin.sgml @@ -85,298 +85,8 @@ - _abstime_ops - abstime[] - - && - <@ - = - @> - - - - _bit_ops - bit[] - - && - <@ - = - @> - - - - _bool_ops - boolean[] - - && - <@ - = - @> - - - - _bpchar_ops - character[] - - && - <@ - = - @> - - - - _bytea_ops - bytea[] - - && - <@ - = - @> - - - - _char_ops - "char"[] - - && - <@ - = - @> - - - - _cidr_ops - cidr[] - - && - <@ - = - @> - - - - _date_ops - date[] - - && - <@ - = - @> - - - - _float4_ops - float4[] - - && - <@ - = - @> - - - - _float8_ops - float8[] - - && - <@ - = - @> - - - - _inet_ops - inet[] - - && - <@ - = - @> - - - - _int2_ops - smallint[] - - && - <@ - = - @> - - - - _int4_ops - integer[] - - && - <@ - = - @> - - - - _int8_ops - bigint[] - - && - <@ - = - @> - - - - _interval_ops - interval[] - - && - <@ - = - @> - - - - _macaddr_ops - macaddr[] - - && - <@ - = - @> - - - - _money_ops - money[] - - && - <@ - = - @> - - - - _name_ops - name[] - - && - <@ - = - @> - - - - _numeric_ops - numeric[] - - && - <@ - = - @> - - - - _oid_ops - oid[] - - && - <@ - = - @> - - - - _oidvector_ops - oidvector[] - - && - <@ - = - @> - - - - _reltime_ops - reltime[] - - && - <@ - = - @> - - - - _text_ops - text[] - - && - <@ - = - @> - - - - _time_ops - time[] - - && - <@ - = - @> - - - - _timestamp_ops - timestamp[] - - && - <@ - = - @> - - - - _timestamptz_ops - timestamp with time zone[] - - && - <@ - = - @> - - - - _timetz_ops - time with time zone[] - - && - <@ - = - @> - - - - _tinterval_ops - tinterval[] - - && - <@ - = - @> - - - - _varbit_ops - bit varying[] - - && - <@ - = - @> - - - - _varchar_ops - character varying[] + array_ops + anyarray && <@ @@ -441,22 +151,10 @@ - There are three methods that an operator class for + There are two methods that an operator class for GIN must provide: - - - int compare(Datum a, Datum b) - - - Compares two keys (not indexed items!) and returns an integer less than - zero, zero, or greater than zero, indicating whether the first key is - less than, equal to, or greater than the second. Null keys are never - passed to this function. - - - - + Datum *extractValue(Datum itemValue, int32 *nkeys, bool **nullFlags) @@ -645,7 +343,38 @@ + + + + In addition, GIN must have a way to sort the key values stored in the index. + The operator class can define the sort ordering by specifying a comparison + method: + + + int compare(Datum a, Datum b) + + + Compares two keys (not indexed items!) and returns an integer less than + zero, zero, or greater than zero, indicating whether the first key is + less than, equal to, or greater than the second. Null keys are never + passed to this function. + + + + + + Alternatively, if the operator class does not provide a compare + method, GIN will look up the default btree operator class for the index + key data type, and use its comparison function. It is recommended to + specify the comparison function in a GIN operator class that is meant for + just one data type, as looking up the btree operator class costs a few + cycles. However, polymorphic GIN operator classes (such + as array_ops) typically cannot specify a single comparison + function. + + + Optionally, an operator class for GIN can supply the following method: @@ -900,11 +629,9 @@ Examples - The PostgreSQL source distribution includes - GIN operator classes for tsvector and - for one-dimensional arrays of all internal types. Prefix searching in - tsvector is implemented using the GIN partial match - feature. + The core PostgreSQL distribution + includes the GIN operator classes previously shown in + . The following contrib modules also contain GIN operator classes: diff --git a/doc/src/sgml/indices.sgml b/doc/src/sgml/indices.sgml index 46f8e55ca9..271c135519 100644 --- a/doc/src/sgml/indices.sgml +++ b/doc/src/sgml/indices.sgml @@ -315,9 +315,8 @@ SELECT * FROM places ORDER BY location <-> point '(101,456)' LIMIT 10; operators with which a GIN index can be used vary depending on the indexing strategy. As an example, the standard distribution of - PostgreSQL includes GIN operator classes - for one-dimensional arrays, which support indexed - queries using these operators: + PostgreSQL includes a GIN operator class + for arrays, which supports indexed queries using these operators: <@ diff --git a/doc/src/sgml/ref/create_opclass.sgml b/doc/src/sgml/ref/create_opclass.sgml index 7b9d55d38d..829d8f2fff 100644 --- a/doc/src/sgml/ref/create_opclass.sgml +++ b/doc/src/sgml/ref/create_opclass.sgml @@ -235,6 +235,11 @@ CREATE OPERATOR CLASS name [ DEFAUL (currently GiST, GIN and BRIN) allow it to be different. The STORAGE clause must be omitted unless the index method allows a different type to be used. + If the column data_type is specified + as anyarray, the storage_type + can be declared as anyelement to indicate that the index + entries are members of the element type belonging to the actual array + type that each particular index is created for. diff --git a/doc/src/sgml/xindex.sgml b/doc/src/sgml/xindex.sgml index f0b711e2ce..333a36c456 100644 --- a/doc/src/sgml/xindex.sgml +++ b/doc/src/sgml/xindex.sgml @@ -288,7 +288,7 @@ have a fixed set of strategies either. Instead the support routines of each operator class interpret the strategy numbers according to the operator class's definition. As an example, the strategy numbers used by - the built-in operator classes for arrays are shown in + the built-in operator class for arrays are shown in . diff --git a/src/backend/access/gin/ginutil.c b/src/backend/access/gin/ginutil.c index d9146488c4..f07eedc0fa 100644 --- a/src/backend/access/gin/ginutil.c +++ b/src/backend/access/gin/ginutil.c @@ -22,7 +22,9 @@ #include "miscadmin.h" #include "storage/indexfsm.h" #include "storage/lmgr.h" +#include "utils/builtins.h" #include "utils/index_selfuncs.h" +#include "utils/typcache.h" /* @@ -105,9 +107,33 @@ initGinState(GinState *state, Relation index) origTupdesc->attrs[i]->attcollation); } - fmgr_info_copy(&(state->compareFn[i]), - index_getprocinfo(index, i + 1, GIN_COMPARE_PROC), - CurrentMemoryContext); + /* + * If the compare proc isn't specified in the opclass definition, look + * up the index key type's default btree comparator. + */ + if (index_getprocid(index, i + 1, GIN_COMPARE_PROC) != InvalidOid) + { + fmgr_info_copy(&(state->compareFn[i]), + index_getprocinfo(index, i + 1, GIN_COMPARE_PROC), + CurrentMemoryContext); + } + else + { + TypeCacheEntry *typentry; + + typentry = lookup_type_cache(origTupdesc->attrs[i]->atttypid, + TYPECACHE_CMP_PROC_FINFO); + if (!OidIsValid(typentry->cmp_proc_finfo.fn_oid)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("could not identify a comparison function for type %s", + format_type_be(origTupdesc->attrs[i]->atttypid)))); + fmgr_info_copy(&(state->compareFn[i]), + &(typentry->cmp_proc_finfo), + CurrentMemoryContext); + } + + /* Opclass must always provide extract procs */ fmgr_info_copy(&(state->extractValueFn[i]), index_getprocinfo(index, i + 1, GIN_EXTRACTVALUE_PROC), CurrentMemoryContext); diff --git a/src/backend/access/gin/ginvalidate.c b/src/backend/access/gin/ginvalidate.c index 032508387d..02196e0f12 100644 --- a/src/backend/access/gin/ginvalidate.c +++ b/src/backend/access/gin/ginvalidate.c @@ -237,7 +237,7 @@ ginvalidate(Oid opclassoid) if (opclassgroup && (opclassgroup->functionset & (((uint64) 1) << i)) != 0) continue; /* got it */ - if (i == GIN_COMPARE_PARTIAL_PROC) + if (i == GIN_COMPARE_PROC || i == GIN_COMPARE_PARTIAL_PROC) continue; /* optional method */ if (i == GIN_CONSISTENT_PROC || i == GIN_TRICONSISTENT_PROC) continue; /* don't need both, see check below loop */ diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index b0b43cf02d..08b646d8f3 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -437,11 +437,28 @@ ConstructTupleDescriptor(Relation heapRelation, keyType = opclassTup->opckeytype; else keyType = amroutine->amkeytype; + + /* + * If keytype is specified as ANYELEMENT, and opcintype is ANYARRAY, + * then the attribute type must be an array (else it'd not have + * matched this opclass); use its element type. + */ + if (keyType == ANYELEMENTOID && opclassTup->opcintype == ANYARRAYOID) + { + keyType = get_base_element_type(to->atttypid); + if (!OidIsValid(keyType)) + elog(ERROR, "could not get element type of array type %u", + to->atttypid); + } + ReleaseSysCache(tuple); + /* + * If a key type different from the heap value is specified, update + * the type-related fields in the index tupdesc. + */ if (OidIsValid(keyType) && keyType != to->atttypid) { - /* index value and heap value have different types */ tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(keyType)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for type %u", keyType); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index ef691c5721..3fdd0d6129 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201609131 +#define CATALOG_VERSION_NO 201609261 #endif diff --git a/src/include/catalog/pg_amop.h b/src/include/catalog/pg_amop.h index 917ed46b71..15b629029f 100644 --- a/src/include/catalog/pg_amop.h +++ b/src/include/catalog/pg_amop.h @@ -673,8 +673,7 @@ DATA(insert ( 2595 718 718 14 s 2864 783 0 )); DATA(insert ( 2595 718 600 15 o 3291 783 1970 )); /* - * gin array_ops (these anyarray operators are used with all the opclasses - * of the family) + * gin array_ops */ DATA(insert ( 2745 2277 2277 1 s 2750 2742 0 )); DATA(insert ( 2745 2277 2277 2 s 2751 2742 0 )); diff --git a/src/include/catalog/pg_amproc.h b/src/include/catalog/pg_amproc.h index 0cbb416392..1b654d5be4 100644 --- a/src/include/catalog/pg_amproc.h +++ b/src/include/catalog/pg_amproc.h @@ -255,156 +255,10 @@ DATA(insert ( 3550 869 869 9 3573 )); /* gin */ -DATA(insert ( 2745 1007 1007 1 351 )); -DATA(insert ( 2745 1007 1007 2 2743 )); -DATA(insert ( 2745 1007 1007 3 2774 )); -DATA(insert ( 2745 1007 1007 4 2744 )); -DATA(insert ( 2745 1007 1007 6 3920 )); -DATA(insert ( 2745 1009 1009 1 360 )); -DATA(insert ( 2745 1009 1009 2 2743 )); -DATA(insert ( 2745 1009 1009 3 2774 )); -DATA(insert ( 2745 1009 1009 4 2744 )); -DATA(insert ( 2745 1009 1009 6 3920 )); -DATA(insert ( 2745 1015 1015 1 360 )); -DATA(insert ( 2745 1015 1015 2 2743 )); -DATA(insert ( 2745 1015 1015 3 2774 )); -DATA(insert ( 2745 1015 1015 4 2744 )); -DATA(insert ( 2745 1015 1015 6 3920 )); -DATA(insert ( 2745 1023 1023 1 357 )); -DATA(insert ( 2745 1023 1023 2 2743 )); -DATA(insert ( 2745 1023 1023 3 2774 )); -DATA(insert ( 2745 1023 1023 4 2744 )); -DATA(insert ( 2745 1023 1023 6 3920 )); -DATA(insert ( 2745 1561 1561 1 1596 )); -DATA(insert ( 2745 1561 1561 2 2743 )); -DATA(insert ( 2745 1561 1561 3 2774 )); -DATA(insert ( 2745 1561 1561 4 2744 )); -DATA(insert ( 2745 1561 1561 6 3920 )); -DATA(insert ( 2745 1000 1000 1 1693 )); -DATA(insert ( 2745 1000 1000 2 2743 )); -DATA(insert ( 2745 1000 1000 3 2774 )); -DATA(insert ( 2745 1000 1000 4 2744 )); -DATA(insert ( 2745 1000 1000 6 3920 )); -DATA(insert ( 2745 1014 1014 1 1078 )); -DATA(insert ( 2745 1014 1014 2 2743 )); -DATA(insert ( 2745 1014 1014 3 2774 )); -DATA(insert ( 2745 1014 1014 4 2744 )); -DATA(insert ( 2745 1014 1014 6 3920 )); -DATA(insert ( 2745 1001 1001 1 1954 )); -DATA(insert ( 2745 1001 1001 2 2743 )); -DATA(insert ( 2745 1001 1001 3 2774 )); -DATA(insert ( 2745 1001 1001 4 2744 )); -DATA(insert ( 2745 1001 1001 6 3920 )); -DATA(insert ( 2745 1002 1002 1 358 )); -DATA(insert ( 2745 1002 1002 2 2743 )); -DATA(insert ( 2745 1002 1002 3 2774 )); -DATA(insert ( 2745 1002 1002 4 2744 )); -DATA(insert ( 2745 1002 1002 6 3920 )); -DATA(insert ( 2745 1182 1182 1 1092 )); -DATA(insert ( 2745 1182 1182 2 2743 )); -DATA(insert ( 2745 1182 1182 3 2774 )); -DATA(insert ( 2745 1182 1182 4 2744 )); -DATA(insert ( 2745 1182 1182 6 3920 )); -DATA(insert ( 2745 1021 1021 1 354 )); -DATA(insert ( 2745 1021 1021 2 2743 )); -DATA(insert ( 2745 1021 1021 3 2774 )); -DATA(insert ( 2745 1021 1021 4 2744 )); -DATA(insert ( 2745 1021 1021 6 3920 )); -DATA(insert ( 2745 1022 1022 1 355 )); -DATA(insert ( 2745 1022 1022 2 2743 )); -DATA(insert ( 2745 1022 1022 3 2774 )); -DATA(insert ( 2745 1022 1022 4 2744 )); -DATA(insert ( 2745 1022 1022 6 3920 )); -DATA(insert ( 2745 1041 1041 1 926 )); -DATA(insert ( 2745 1041 1041 2 2743 )); -DATA(insert ( 2745 1041 1041 3 2774 )); -DATA(insert ( 2745 1041 1041 4 2744 )); -DATA(insert ( 2745 1041 1041 6 3920 )); -DATA(insert ( 2745 651 651 1 926 )); -DATA(insert ( 2745 651 651 2 2743 )); -DATA(insert ( 2745 651 651 3 2774 )); -DATA(insert ( 2745 651 651 4 2744 )); -DATA(insert ( 2745 651 651 6 3920 )); -DATA(insert ( 2745 1005 1005 1 350 )); -DATA(insert ( 2745 1005 1005 2 2743 )); -DATA(insert ( 2745 1005 1005 3 2774 )); -DATA(insert ( 2745 1005 1005 4 2744 )); -DATA(insert ( 2745 1005 1005 6 3920 )); -DATA(insert ( 2745 1016 1016 1 842 )); -DATA(insert ( 2745 1016 1016 2 2743 )); -DATA(insert ( 2745 1016 1016 3 2774 )); -DATA(insert ( 2745 1016 1016 4 2744 )); -DATA(insert ( 2745 1016 1016 6 3920 )); -DATA(insert ( 2745 1187 1187 1 1315 )); -DATA(insert ( 2745 1187 1187 2 2743 )); -DATA(insert ( 2745 1187 1187 3 2774 )); -DATA(insert ( 2745 1187 1187 4 2744 )); -DATA(insert ( 2745 1187 1187 6 3920 )); -DATA(insert ( 2745 1040 1040 1 836 )); -DATA(insert ( 2745 1040 1040 2 2743 )); -DATA(insert ( 2745 1040 1040 3 2774 )); -DATA(insert ( 2745 1040 1040 4 2744 )); -DATA(insert ( 2745 1040 1040 6 3920 )); -DATA(insert ( 2745 1003 1003 1 359 )); -DATA(insert ( 2745 1003 1003 2 2743 )); -DATA(insert ( 2745 1003 1003 3 2774 )); -DATA(insert ( 2745 1003 1003 4 2744 )); -DATA(insert ( 2745 1003 1003 6 3920 )); -DATA(insert ( 2745 1231 1231 1 1769 )); -DATA(insert ( 2745 1231 1231 2 2743 )); -DATA(insert ( 2745 1231 1231 3 2774 )); -DATA(insert ( 2745 1231 1231 4 2744 )); -DATA(insert ( 2745 1231 1231 6 3920 )); -DATA(insert ( 2745 1028 1028 1 356 )); -DATA(insert ( 2745 1028 1028 2 2743 )); -DATA(insert ( 2745 1028 1028 3 2774 )); -DATA(insert ( 2745 1028 1028 4 2744 )); -DATA(insert ( 2745 1028 1028 6 3920 )); -DATA(insert ( 2745 1013 1013 1 404 )); -DATA(insert ( 2745 1013 1013 2 2743 )); -DATA(insert ( 2745 1013 1013 3 2774 )); -DATA(insert ( 2745 1013 1013 4 2744 )); -DATA(insert ( 2745 1013 1013 6 3920 )); -DATA(insert ( 2745 1183 1183 1 1107 )); -DATA(insert ( 2745 1183 1183 2 2743 )); -DATA(insert ( 2745 1183 1183 3 2774 )); -DATA(insert ( 2745 1183 1183 4 2744 )); -DATA(insert ( 2745 1183 1183 6 3920 )); -DATA(insert ( 2745 1185 1185 1 1314 )); -DATA(insert ( 2745 1185 1185 2 2743 )); -DATA(insert ( 2745 1185 1185 3 2774 )); -DATA(insert ( 2745 1185 1185 4 2744 )); -DATA(insert ( 2745 1185 1185 6 3920 )); -DATA(insert ( 2745 1270 1270 1 1358 )); -DATA(insert ( 2745 1270 1270 2 2743 )); -DATA(insert ( 2745 1270 1270 3 2774 )); -DATA(insert ( 2745 1270 1270 4 2744 )); -DATA(insert ( 2745 1270 1270 6 3920 )); -DATA(insert ( 2745 1563 1563 1 1672 )); -DATA(insert ( 2745 1563 1563 2 2743 )); -DATA(insert ( 2745 1563 1563 3 2774 )); -DATA(insert ( 2745 1563 1563 4 2744 )); -DATA(insert ( 2745 1563 1563 6 3920 )); -DATA(insert ( 2745 1115 1115 1 2045 )); -DATA(insert ( 2745 1115 1115 2 2743 )); -DATA(insert ( 2745 1115 1115 3 2774 )); -DATA(insert ( 2745 1115 1115 4 2744 )); -DATA(insert ( 2745 1115 1115 6 3920 )); -DATA(insert ( 2745 791 791 1 377 )); -DATA(insert ( 2745 791 791 2 2743 )); -DATA(insert ( 2745 791 791 3 2774 )); -DATA(insert ( 2745 791 791 4 2744 )); -DATA(insert ( 2745 791 791 6 3920 )); -DATA(insert ( 2745 1024 1024 1 380 )); -DATA(insert ( 2745 1024 1024 2 2743 )); -DATA(insert ( 2745 1024 1024 3 2774 )); -DATA(insert ( 2745 1024 1024 4 2744 )); -DATA(insert ( 2745 1024 1024 6 3920 )); -DATA(insert ( 2745 1025 1025 1 381 )); -DATA(insert ( 2745 1025 1025 2 2743 )); -DATA(insert ( 2745 1025 1025 3 2774 )); -DATA(insert ( 2745 1025 1025 4 2744 )); -DATA(insert ( 2745 1025 1025 6 3920 )); +DATA(insert ( 2745 2277 2277 2 2743 )); +DATA(insert ( 2745 2277 2277 3 2774 )); +DATA(insert ( 2745 2277 2277 4 2744 )); +DATA(insert ( 2745 2277 2277 6 3920 )); DATA(insert ( 3659 3614 3614 1 3724 )); DATA(insert ( 3659 3614 3614 2 3656 )); DATA(insert ( 3659 3614 3614 3 3657 )); diff --git a/src/include/catalog/pg_opclass.h b/src/include/catalog/pg_opclass.h index f40b06112b..5900cdc5b0 100644 --- a/src/include/catalog/pg_opclass.h +++ b/src/include/catalog/pg_opclass.h @@ -184,36 +184,7 @@ DATA(insert ( 783 box_ops PGNSP PGUID 2593 603 t 0 )); DATA(insert ( 783 point_ops PGNSP PGUID 1029 600 t 603 )); DATA(insert ( 783 poly_ops PGNSP PGUID 2594 604 t 603 )); DATA(insert ( 783 circle_ops PGNSP PGUID 2595 718 t 603 )); -DATA(insert ( 2742 _int4_ops PGNSP PGUID 2745 1007 t 23 )); -DATA(insert ( 2742 _text_ops PGNSP PGUID 2745 1009 t 25 )); -DATA(insert ( 2742 _abstime_ops PGNSP PGUID 2745 1023 t 702 )); -DATA(insert ( 2742 _bit_ops PGNSP PGUID 2745 1561 t 1560 )); -DATA(insert ( 2742 _bool_ops PGNSP PGUID 2745 1000 t 16 )); -DATA(insert ( 2742 _bpchar_ops PGNSP PGUID 2745 1014 t 1042 )); -DATA(insert ( 2742 _bytea_ops PGNSP PGUID 2745 1001 t 17 )); -DATA(insert ( 2742 _char_ops PGNSP PGUID 2745 1002 t 18 )); -DATA(insert ( 2742 _cidr_ops PGNSP PGUID 2745 651 t 650 )); -DATA(insert ( 2742 _date_ops PGNSP PGUID 2745 1182 t 1082 )); -DATA(insert ( 2742 _float4_ops PGNSP PGUID 2745 1021 t 700 )); -DATA(insert ( 2742 _float8_ops PGNSP PGUID 2745 1022 t 701 )); -DATA(insert ( 2742 _inet_ops PGNSP PGUID 2745 1041 t 869 )); -DATA(insert ( 2742 _int2_ops PGNSP PGUID 2745 1005 t 21 )); -DATA(insert ( 2742 _int8_ops PGNSP PGUID 2745 1016 t 20 )); -DATA(insert ( 2742 _interval_ops PGNSP PGUID 2745 1187 t 1186 )); -DATA(insert ( 2742 _macaddr_ops PGNSP PGUID 2745 1040 t 829 )); -DATA(insert ( 2742 _name_ops PGNSP PGUID 2745 1003 t 19 )); -DATA(insert ( 2742 _numeric_ops PGNSP PGUID 2745 1231 t 1700 )); -DATA(insert ( 2742 _oid_ops PGNSP PGUID 2745 1028 t 26 )); -DATA(insert ( 2742 _oidvector_ops PGNSP PGUID 2745 1013 t 30 )); -DATA(insert ( 2742 _time_ops PGNSP PGUID 2745 1183 t 1083 )); -DATA(insert ( 2742 _timestamptz_ops PGNSP PGUID 2745 1185 t 1184 )); -DATA(insert ( 2742 _timetz_ops PGNSP PGUID 2745 1270 t 1266 )); -DATA(insert ( 2742 _varbit_ops PGNSP PGUID 2745 1563 t 1562 )); -DATA(insert ( 2742 _varchar_ops PGNSP PGUID 2745 1015 t 1043 )); -DATA(insert ( 2742 _timestamp_ops PGNSP PGUID 2745 1115 t 1114 )); -DATA(insert ( 2742 _money_ops PGNSP PGUID 2745 791 t 790 )); -DATA(insert ( 2742 _reltime_ops PGNSP PGUID 2745 1024 t 703 )); -DATA(insert ( 2742 _tinterval_ops PGNSP PGUID 2745 1025 t 704 )); +DATA(insert ( 2742 array_ops PGNSP PGUID 2745 2277 t 2283 )); DATA(insert ( 403 uuid_ops PGNSP PGUID 2968 2950 t 0 )); DATA(insert ( 405 uuid_ops PGNSP PGUID 2969 2950 t 0 )); DATA(insert ( 403 pg_lsn_ops PGNSP PGUID 3253 3220 t 0 )); From 9779bda86c026e645773a3308f9169c7c0791f7c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 26 Sep 2016 20:23:50 -0400 Subject: [PATCH 220/871] Fix newly-introduced issues in pgbench. The result of FD_ISSET() doesn't necessarily fit in a bool, though assigning it to one might accidentally work depending on platform and which socket FD number is being inquired of. Rewrite to test it with if(), rather than making any specific assumption about the result width, to match the way every other such call in PG is written. Don't break out of the input_mask-filling loop after finding the first client that we're waiting for results from. That mostly breaks parallel query management. Also, if we choose not to call select(), be sure to clear out any bits the mask-filling loop might have set, so that we don't accidentally call doCustom for clients we don't know have input. Doing so would likely be harmless, but it's a waste of cycles and doesn't seem to be intended. Make this_usec wide enough. (Yeah, the value would usually fit in an int, but then why are we using int64 everywhere else?) Minor cosmetic adjustments, mostly comment improvements. Problems introduced by commit 12788ae49. The first issue was discovered by buildfarm testing, the others by code review. --- src/bin/pgbench/pgbench.c | 82 +++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 1fb4ae46d5..d44cfdab49 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -299,7 +299,7 @@ typedef enum */ CSTATE_ABORTED, CSTATE_FINISHED -} ConnectionStateEnum; +} ConnectionStateEnum; /* * Connection state. @@ -4420,43 +4420,43 @@ threadRun(void *arg) initStats(&aggs, INSTR_TIME_GET_DOUBLE(thread->start_time)); last = aggs; - /* initialize explicitely the state machines */ + /* explicitly initialize the state machines */ for (i = 0; i < nstate; i++) { state[i].state = CSTATE_CHOOSE_SCRIPT; } + /* loop till all clients have terminated */ while (remains > 0) { fd_set input_mask; - int maxsock; /* max socket number to be waited */ - int64 now_usec = 0; + int maxsock; /* max socket number to be waited for */ int64 min_usec; + int64 now_usec = 0; /* set this only if needed */ + /* identify which client sockets should be checked for input */ FD_ZERO(&input_mask); - maxsock = -1; min_usec = PG_INT64_MAX; for (i = 0; i < nstate; i++) { CState *st = &state[i]; - int sock; if (st->state == CSTATE_THROTTLE && timer_exceeded) { - /* interrupt client which has not started a transaction */ + /* interrupt client that has not started a transaction */ st->state = CSTATE_FINISHED; - remains--; PQfinish(st->con); st->con = NULL; - continue; + remains--; } else if (st->state == CSTATE_SLEEP || st->state == CSTATE_THROTTLE) { /* a nap from the script, or under throttling */ - int this_usec; + int64 this_usec; - if (min_usec == PG_INT64_MAX) + /* get current time if needed */ + if (now_usec == 0) { instr_time now; @@ -4464,6 +4464,7 @@ threadRun(void *arg) now_usec = INSTR_TIME_GET_MICROSEC(now); } + /* min_usec should be the minimum delay across all clients */ this_usec = (st->state == CSTATE_SLEEP ? st->sleep_until : st->txn_scheduled) - now_usec; if (min_usec > this_usec) @@ -4475,22 +4476,26 @@ threadRun(void *arg) * waiting for result from server - nothing to do unless the * socket is readable */ - sock = PQsocket(st->con); + int sock = PQsocket(st->con); + if (sock < 0) { - fprintf(stderr, "invalid socket: %s", PQerrorMessage(st->con)); + fprintf(stderr, "invalid socket: %s", + PQerrorMessage(st->con)); goto done; } FD_SET(sock, &input_mask); - if (maxsock < sock) maxsock = sock; - break; } - else if (st->state != CSTATE_ABORTED && st->state != CSTATE_FINISHED) + else if (st->state != CSTATE_ABORTED && + st->state != CSTATE_FINISHED) { - /* the connection is ready to run */ + /* + * This client thread is ready to do something, so we don't + * want to wait. No need to examine additional clients. + */ min_usec = 0; break; } @@ -4515,9 +4520,10 @@ threadRun(void *arg) } /* - * Sleep until we receive data from the server, or a nap-time - * specified in the script ends, or it's time to print a progress - * report. + * If no clients are ready to execute actions, sleep until we receive + * data from the server, or a nap-time specified in the script ends, + * or it's time to print a progress report. Update input_mask to show + * which client(s) received data. */ if (min_usec > 0 && maxsock != -1) { @@ -4536,21 +4542,29 @@ threadRun(void *arg) if (nsocks < 0) { if (errno == EINTR) + { + /* On EINTR, go back to top of loop */ continue; + } /* must be something wrong */ fprintf(stderr, "select() failed: %s\n", strerror(errno)); goto done; } } + else + { + /* If we didn't call select(), don't try to read any data */ + FD_ZERO(&input_mask); + } /* ok, advance the state machine of each connection */ for (i = 0; i < nstate; i++) { CState *st = &state[i]; - bool ready; - if (st->state == CSTATE_WAIT_RESULT && st->con) + if (st->state == CSTATE_WAIT_RESULT) { + /* don't call doCustom unless data is available */ int sock = PQsocket(st->con); if (sock < 0) @@ -4560,22 +4574,24 @@ threadRun(void *arg) goto done; } - ready = FD_ISSET(sock, &input_mask); + if (!FD_ISSET(sock, &input_mask)) + continue; } - else if (st->state == CSTATE_FINISHED || st->state == CSTATE_ABORTED) - ready = false; - else - ready = true; - - if (ready) + else if (st->state == CSTATE_FINISHED || + st->state == CSTATE_ABORTED) { - doCustom(thread, st, &aggs); - if (st->state == CSTATE_FINISHED || st->state == CSTATE_ABORTED) - remains--; + /* this client is done, no need to consider it anymore */ + continue; } + + doCustom(thread, st, &aggs); + + /* If doCustom changed client to finished state, reduce remains */ + if (st->state == CSTATE_FINISHED || st->state == CSTATE_ABORTED) + remains--; } - /* progress report by thread 0 for all threads */ + /* progress report is made by thread 0 for all threads */ if (progress && thread->tid == 0) { instr_time now_time; From 440c8d1bbc8c62d225ab7c3e30e0a7db2639cd0f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 26 Sep 2016 12:00:00 -0400 Subject: [PATCH 221/871] Fix some typos in comment --- src/include/access/hash.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/access/hash.h b/src/include/access/hash.h index d9df904555..491d4c90bb 100644 --- a/src/include/access/hash.h +++ b/src/include/access/hash.h @@ -119,7 +119,7 @@ typedef HashScanOpaqueData *HashScanOpaque; #define HASH_VERSION 2 /* 2 signifies only hash key value is stored */ /* - * Spares[] holds the number of overflow pages currently allocated at or + * spares[] holds the number of overflow pages currently allocated at or * before a certain splitpoint. For example, if spares[3] = 7 then there are * 7 ovflpages before splitpoint 3 (compare BUCKET_TO_BLKNO macro). The * value in spares[ovflpoint] increases as overflow pages are added at the @@ -129,14 +129,14 @@ typedef HashScanOpaqueData *HashScanOpaque; * * ovflpages that have been recycled for reuse can be found by looking at * bitmaps that are stored within ovflpages dedicated for the purpose. - * The blknos of these bitmap pages are kept in bitmaps[]; nmaps is the + * The blknos of these bitmap pages are kept in mapp[]; nmaps is the * number of currently existing bitmaps. * * The limitation on the size of spares[] comes from the fact that there's * no point in having more than 2^32 buckets with only uint32 hashcodes. * There is no particular upper limit on the size of mapp[], other than * needing to fit into the metapage. (With 8K block size, 128 bitmaps - * limit us to 64 Gb of overflow space...) + * limit us to 64 GB of overflow space...) */ #define HASH_MAX_SPLITPOINTS 32 #define HASH_MAX_BITMAPS 128 From 51c3e9fade76c12e4aa37bffdf800bbf74fb3fb1 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 27 Sep 2016 01:05:21 -0300 Subject: [PATCH 222/871] Include where needed is required by POSIX.1-2001 to get the prototype of select(2), but nearly no systems enforce that because older standards let you get away with including some other headers. Recent OpenBSD hacking has removed that frail touch of friendliness, however, which broke some compiles; fix all the way back to 9.1 by adding the required standard. Only vacuumdb.c was reported to fail, but it seems easier to fix the whole lot in a fell swoop. Per bug #14334 by Sean Farrell. --- src/backend/libpq/auth.c | 3 +++ src/backend/postmaster/pgstat.c | 3 +++ src/bin/pg_basebackup/pg_basebackup.c | 4 +++- src/bin/pg_basebackup/pg_recvlogical.c | 3 +++ src/bin/pg_basebackup/receivelog.c | 3 +++ src/bin/pg_dump/parallel.c | 4 ++++ src/bin/scripts/vacuumdb.c | 4 ++++ src/port/pgsleep.c | 3 +++ src/test/examples/testlibpq2.c | 4 ++++ 9 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index d907e6bc91..0ba8530114 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -20,6 +20,9 @@ #include #include #include +#ifdef HAVE_SYS_SELECT_H +#include +#endif #include "common/ip.h" #include "common/md5.h" diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 8a2ce91344..96578dcedb 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -28,6 +28,9 @@ #include #include #include +#ifdef HAVE_SYS_SELECT_H +#include +#endif #include "pgstat.h" diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 42f3b273a6..d077544d62 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -20,7 +20,9 @@ #include #include #include - +#ifdef HAVE_SYS_SELECT_H +#include +#endif #ifdef HAVE_LIBZ #include #endif diff --git a/src/bin/pg_basebackup/pg_recvlogical.c b/src/bin/pg_basebackup/pg_recvlogical.c index 4c6cf7054e..cb5f989a76 100644 --- a/src/bin/pg_basebackup/pg_recvlogical.c +++ b/src/bin/pg_basebackup/pg_recvlogical.c @@ -15,6 +15,9 @@ #include #include #include +#ifdef HAVE_SYS_SELECT_H +#include +#endif /* local includes */ #include "streamutil.h" diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index 062730b6b4..3a921ebf2d 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -16,6 +16,9 @@ #include #include +#ifdef HAVE_SYS_SELECT_H +#include +#endif /* local includes */ #include "receivelog.h" diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index 4549d11e20..bfd023f3e1 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -59,6 +59,10 @@ #include "postgres_fe.h" +#ifdef HAVE_SYS_SELECT_H +#include +#endif + #include "parallel.h" #include "pg_backup_utils.h" #include "fe_utils/string_utils.h" diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index c10b58bf0f..32cb0fca2f 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -12,6 +12,10 @@ #include "postgres_fe.h" +#ifdef HAVE_SYS_SELECT_H +#include +#endif + #include "common.h" #include "fe_utils/simple_list.h" #include "fe_utils/string_utils.h" diff --git a/src/port/pgsleep.c b/src/port/pgsleep.c index ef7f7aba6b..ecefe04ec0 100644 --- a/src/port/pgsleep.c +++ b/src/port/pgsleep.c @@ -14,6 +14,9 @@ #include #include +#ifdef HAVE_SYS_SELECT_H +#include +#endif /* * In a Windows backend, we don't use this implementation, but rather diff --git a/src/test/examples/testlibpq2.c b/src/test/examples/testlibpq2.c index 850993f6e8..07c6317a21 100644 --- a/src/test/examples/testlibpq2.c +++ b/src/test/examples/testlibpq2.c @@ -34,6 +34,10 @@ #include #include #include +#ifdef HAVE_SYS_SELECT_H +#include +#endif + #include "libpq-fe.h" static void From f31a931fade868d788ef4480c59753a2d5059246 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 27 Sep 2016 11:38:33 -0400 Subject: [PATCH 223/871] Improve contrib/cube's handling of zero-D cubes, infinities, and NaNs. It's always been possible to create a zero-dimensional cube by converting from a zero-length float8 array, but cube_in failed to accept the '()' representation that cube_out produced for that case, resulting in a dump/reload hazard. Make it accept the case. Also fix a couple of other places that didn't behave sanely for zero-dimensional cubes: cube_size would produce 1.0 when surely the answer should be 0.0, and g_cube_distance risked a divide-by-zero failure. Likewise, it's always been possible to create cubes containing float8 infinity or NaN coordinate values, but cube_in couldn't parse such input, and cube_out produced platform-dependent spellings of the values. Convert them to use float8in_internal and float8out_internal so that the behavior will be the same as for float8, as we recently did for the core geometric types (cf commit 50861cd68). As in that commit, I don't pretend that this patch fixes all insane corner-case behaviors that may exist for NaNs, but it's a step forward. (This change allows removal of the separate cube_1.out and cube_3.out expected-files, as the platform dependency that previously required them is now gone: an underflowing coordinate value will now produce an error not plus or minus zero.) Make errors from cube_in follow project conventions as to spelling ("invalid input syntax for cube" not "bad cube representation") and errcode (INVALID_TEXT_REPRESENTATION not SYNTAX_ERROR). Also a few marginal code cleanups and comment improvements. Tom Lane, reviewed by Amul Sul Discussion: <15085.1472494782@sss.pgh.pa.us> --- contrib/cube/cube.c | 70 +- contrib/cube/cubedata.h | 7 + contrib/cube/cubeparse.y | 148 +-- contrib/cube/cubescan.l | 15 +- contrib/cube/expected/cube.out | 115 +- contrib/cube/expected/cube_1.out | 1710 ------------------------------ contrib/cube/expected/cube_2.out | 115 +- contrib/cube/expected/cube_3.out | 1710 ------------------------------ contrib/cube/sql/cube.sql | 11 +- 9 files changed, 275 insertions(+), 3626 deletions(-) delete mode 100644 contrib/cube/expected/cube_1.out delete mode 100644 contrib/cube/expected/cube_3.out diff --git a/contrib/cube/cube.c b/contrib/cube/cube.c index 3feddef8f3..2bb2ed029d 100644 --- a/contrib/cube/cube.c +++ b/contrib/cube/cube.c @@ -122,7 +122,7 @@ cube_in(PG_FUNCTION_ARGS) cube_scanner_init(str); if (cube_yyparse(&result) != 0) - cube_yyerror(&result, "bogus input"); + cube_yyerror(&result, "cube parser failed"); cube_scanner_finish(); @@ -254,12 +254,9 @@ cube_subset(PG_FUNCTION_ARGS) for (i = 0; i < dim; i++) { if ((dx[i] <= 0) || (dx[i] > DIM(c))) - { - pfree(result); ereport(ERROR, (errcode(ERRCODE_ARRAY_ELEMENT_ERROR), errmsg("Index out of bounds"))); - } result->x[i] = c->x[dx[i] - 1]; if (!IS_POINT(c)) result->x[i + dim] = c->x[dx[i] + DIM(c) - 1]; @@ -276,27 +273,15 @@ cube_out(PG_FUNCTION_ARGS) StringInfoData buf; int dim = DIM(cube); int i; - int ndig; initStringInfo(&buf); - /* - * Get the number of digits to display. - */ - ndig = DBL_DIG + extra_float_digits; - if (ndig < 1) - ndig = 1; - - /* - * while printing the first (LL) corner, check if it is equal to the - * second one - */ appendStringInfoChar(&buf, '('); for (i = 0; i < dim; i++) { if (i > 0) appendStringInfoString(&buf, ", "); - appendStringInfo(&buf, "%.*g", ndig, LL_COORD(cube, i)); + appendStringInfoString(&buf, float8out_internal(LL_COORD(cube, i))); } appendStringInfoChar(&buf, ')'); @@ -307,7 +292,7 @@ cube_out(PG_FUNCTION_ARGS) { if (i > 0) appendStringInfoString(&buf, ", "); - appendStringInfo(&buf, "%.*g", ndig, UR_COORD(cube, i)); + appendStringInfoString(&buf, float8out_internal(UR_COORD(cube, i))); } appendStringInfoChar(&buf, ')'); } @@ -370,9 +355,6 @@ g_cube_union(PG_FUNCTION_ARGS) NDBOX *tmp; int i; - /* - * fprintf(stderr, "union\n"); - */ tmp = DatumGetNDBOX(entryvec->vector[0].key); /* @@ -441,9 +423,6 @@ g_cube_penalty(PG_FUNCTION_ARGS) rt_cube_size(DatumGetNDBOX(origentry->key), &tmp2); *result = (float) (tmp1 - tmp2); - /* - * fprintf(stderr, "penalty\n"); fprintf(stderr, "\t%g\n", *result); - */ PG_RETURN_FLOAT8(*result); } @@ -484,9 +463,6 @@ g_cube_picksplit(PG_FUNCTION_ARGS) *right; OffsetNumber maxoff; - /* - * fprintf(stderr, "picksplit\n"); - */ maxoff = entryvec->n - 2; nbytes = (maxoff + 2) * sizeof(OffsetNumber); v->spl_left = (OffsetNumber *) palloc(nbytes); @@ -617,9 +593,6 @@ g_cube_same(PG_FUNCTION_ARGS) else *result = FALSE; - /* - * fprintf(stderr, "same: %s\n", (*result ? "TRUE" : "FALSE" )); - */ PG_RETURN_NDBOX(result); } @@ -633,9 +606,6 @@ g_cube_leaf_consistent(NDBOX *key, { bool retval; - /* - * fprintf(stderr, "leaf_consistent, %d\n", strategy); - */ switch (strategy) { case RTOverlapStrategyNumber: @@ -665,9 +635,6 @@ g_cube_internal_consistent(NDBOX *key, { bool retval; - /* - * fprintf(stderr, "internal_consistent, %d\n", strategy); - */ switch (strategy) { case RTOverlapStrategyNumber: @@ -865,12 +832,8 @@ cube_size(PG_FUNCTION_ARGS) { NDBOX *a = PG_GETARG_NDBOX(0); double result; - int i; - - result = 1.0; - for (i = 0; i < DIM(a); i++) - result = result * Abs((LL_COORD(a, i) - UR_COORD(a, i))); + rt_cube_size(a, &result); PG_FREE_IF_COPY(a, 0); PG_RETURN_FLOAT8(result); } @@ -878,17 +841,26 @@ cube_size(PG_FUNCTION_ARGS) void rt_cube_size(NDBOX *a, double *size) { + double result; int i; if (a == (NDBOX *) NULL) - *size = 0.0; + { + /* special case for GiST */ + result = 0.0; + } + else if (IS_POINT(a) || DIM(a) == 0) + { + /* necessarily has zero size */ + result = 0.0; + } else { - *size = 1.0; + result = 1.0; for (i = 0; i < DIM(a); i++) - *size = (*size) * Abs(UR_COORD(a, i) - LL_COORD(a, i)); + result *= Abs(UR_COORD(a, i) - LL_COORD(a, i)); } - return; + *size = result; } /* make up a metric in which one box will be 'lower' than the other @@ -1155,10 +1127,6 @@ cube_overlap_v0(NDBOX *a, NDBOX *b) { int i; - /* - * This *very bad* error was found in the source: if ( (a==NULL) || - * (b=NULL) ) return(FALSE); - */ if ((a == NULL) || (b == NULL)) return (FALSE); @@ -1370,7 +1338,9 @@ g_cube_distance(PG_FUNCTION_ARGS) { int coord = PG_GETARG_INT32(1); - if (IS_POINT(cube)) + if (DIM(cube) == 0) + retval = 0.0; + else if (IS_POINT(cube)) retval = cube->x[(coord - 1) % DIM(cube)]; else retval = Min(cube->x[(coord - 1) % DIM(cube)], diff --git a/contrib/cube/cubedata.h b/contrib/cube/cubedata.h index 7eaac39640..af02464178 100644 --- a/contrib/cube/cubedata.h +++ b/contrib/cube/cubedata.h @@ -1,5 +1,9 @@ /* contrib/cube/cubedata.h */ +/* + * This limit is pretty arbitrary, but don't make it so large that you + * risk overflow in sizing calculations. + */ #define CUBE_MAX_DIM (100) typedef struct NDBOX @@ -29,6 +33,7 @@ typedef struct NDBOX double x[FLEXIBLE_ARRAY_MEMBER]; } NDBOX; +/* NDBOX access macros */ #define POINT_BIT 0x80000000 #define DIM_MASK 0x7fffffff @@ -43,10 +48,12 @@ typedef struct NDBOX #define POINT_SIZE(_dim) (offsetof(NDBOX, x) + sizeof(double)*(_dim)) #define CUBE_SIZE(_dim) (offsetof(NDBOX, x) + sizeof(double)*(_dim)*2) +/* fmgr interface macros */ #define DatumGetNDBOX(x) ((NDBOX *) PG_DETOAST_DATUM(x)) #define PG_GETARG_NDBOX(x) DatumGetNDBOX(PG_GETARG_DATUM(x)) #define PG_RETURN_NDBOX(x) PG_RETURN_POINTER(x) +/* GiST operator strategy numbers */ #define CubeKNNDistanceCoord 15 /* ~> */ #define CubeKNNDistanceTaxicab 16 /* <#> */ #define CubeKNNDistanceEuclid 17 /* <-> */ diff --git a/contrib/cube/cubeparse.y b/contrib/cube/cubeparse.y index 33606c741c..1b65fa967c 100644 --- a/contrib/cube/cubeparse.y +++ b/contrib/cube/cubeparse.y @@ -4,12 +4,13 @@ /* NdBox = [(lowerleft),(upperright)] */ /* [(xLL(1)...xLL(N)),(xUR(1)...xUR(n))] */ -#define YYSTYPE char * -#define YYDEBUG 1 - #include "postgres.h" #include "cubedata.h" +#include "utils/builtins.h" + +/* All grammar constructs return strings */ +#define YYSTYPE char * /* * Bison doesn't allocate anything that needs to live across parser calls, @@ -25,9 +26,9 @@ static char *scanbuf; static int scanbuflen; -static int delim_count(char *s, char delim); -static NDBOX * write_box(unsigned int dim, char *str1, char *str2); -static NDBOX * write_point_as_box(char *s, int dim); +static int item_count(const char *s, char delim); +static NDBOX *write_box(int dim, char *str1, char *str2); +static NDBOX *write_point_as_box(int dim, char *str); %} @@ -46,47 +47,48 @@ box: O_BRACKET paren_list COMMA paren_list C_BRACKET { int dim; - dim = delim_count($2, ',') + 1; - if ((delim_count($4, ',') + 1) != dim) + dim = item_count($2, ','); + if (item_count($4, ',') != dim) { ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("bad cube representation"), + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for cube"), errdetail("Different point dimensions in (%s) and (%s).", $2, $4))); YYABORT; } - if (dim > CUBE_MAX_DIM) { + if (dim > CUBE_MAX_DIM) + { ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("bad cube representation"), + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for cube"), errdetail("A cube cannot have more than %d dimensions.", CUBE_MAX_DIM))); YYABORT; } *result = write_box( dim, $2, $4 ); - } | paren_list COMMA paren_list { int dim; - dim = delim_count($1, ',') + 1; - - if ( (delim_count($3, ',') + 1) != dim ) { + dim = item_count($1, ','); + if (item_count($3, ',') != dim) + { ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("bad cube representation"), + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for cube"), errdetail("Different point dimensions in (%s) and (%s).", $1, $3))); YYABORT; } - if (dim > CUBE_MAX_DIM) { + if (dim > CUBE_MAX_DIM) + { ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("bad cube representation"), + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for cube"), errdetail("A cube cannot have more than %d dimensions.", CUBE_MAX_DIM))); YYABORT; @@ -99,33 +101,36 @@ box: O_BRACKET paren_list COMMA paren_list C_BRACKET { int dim; - dim = delim_count($1, ',') + 1; - if (dim > CUBE_MAX_DIM) { + dim = item_count($1, ','); + if (dim > CUBE_MAX_DIM) + { ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("bad cube representation"), + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for cube"), errdetail("A cube cannot have more than %d dimensions.", CUBE_MAX_DIM))); YYABORT; } - *result = write_point_as_box($1, dim); + *result = write_point_as_box(dim, $1); } | list { int dim; - dim = delim_count($1, ',') + 1; - if (dim > CUBE_MAX_DIM) { + dim = item_count($1, ','); + if (dim > CUBE_MAX_DIM) + { ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("bad cube representation"), + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for cube"), errdetail("A cube cannot have more than %d dimensions.", CUBE_MAX_DIM))); YYABORT; } - *result = write_point_as_box($1, dim); + + *result = write_point_as_box(dim, $1); } ; @@ -133,6 +138,10 @@ paren_list: O_PAREN list C_PAREN { $$ = $2; } + | O_PAREN C_PAREN + { + $$ = pstrdup(""); + } ; list: CUBEFLOAT @@ -151,24 +160,30 @@ list: CUBEFLOAT %% +/* This assumes the string has been normalized by productions above */ static int -delim_count(char *s, char delim) +item_count(const char *s, char delim) { - int ndelim = 0; + int nitems = 0; - while ((s = strchr(s, delim)) != NULL) + if (s[0] != '\0') { - ndelim++; - s++; + nitems++; + while ((s = strchr(s, delim)) != NULL) + { + nitems++; + s++; + } } - return (ndelim); + return nitems; } static NDBOX * -write_box(unsigned int dim, char *str1, char *str2) +write_box(int dim, char *str1, char *str2) { NDBOX *bp; char *s; + char *endptr; int i; int size = CUBE_SIZE(dim); bool point = true; @@ -178,50 +193,58 @@ write_box(unsigned int dim, char *str1, char *str2) SET_DIM(bp, dim); s = str1; - bp->x[i=0] = strtod(s, NULL); + i = 0; + if (dim > 0) + bp->x[i++] = float8in_internal(s, &endptr, "cube", str1); while ((s = strchr(s, ',')) != NULL) { - s++; i++; - bp->x[i] = strtod(s, NULL); + s++; + bp->x[i++] = float8in_internal(s, &endptr, "cube", str1); } + Assert(i == dim); s = str2; - bp->x[i=dim] = strtod(s, NULL); - if (bp->x[dim] != bp->x[0]) - point = false; + if (dim > 0) + { + bp->x[i] = float8in_internal(s, &endptr, "cube", str2); + /* code this way to do right thing with NaN */ + point &= (bp->x[i] == bp->x[0]); + i++; + } while ((s = strchr(s, ',')) != NULL) { - s++; i++; - bp->x[i] = strtod(s, NULL); - if (bp->x[i] != bp->x[i-dim]) - point = false; + s++; + bp->x[i] = float8in_internal(s, &endptr, "cube", str2); + point &= (bp->x[i] == bp->x[i - dim]); + i++; } + Assert(i == dim * 2); if (point) { /* * The value turned out to be a point, ie. all the upper-right * coordinates were equal to the lower-left coordinates. Resize the - * the cube we constructed. Note: we don't bother to repalloc() it - * smaller, it's unlikely that the tiny amount of memory free'd that - * way would be useful. + * cube we constructed. Note: we don't bother to repalloc() it + * smaller, as it's unlikely that the tiny amount of memory freed + * that way would be useful, and the output is always short-lived. */ size = POINT_SIZE(dim); SET_VARSIZE(bp, size); SET_POINT_BIT(bp); } - return(bp); + return bp; } static NDBOX * -write_point_as_box(char *str, int dim) +write_point_as_box(int dim, char *str) { NDBOX *bp; int i, size; - double x; - char *s = str; + char *s; + char *endptr; size = POINT_SIZE(dim); bp = palloc0(size); @@ -229,17 +252,18 @@ write_point_as_box(char *str, int dim) SET_DIM(bp, dim); SET_POINT_BIT(bp); + s = str; i = 0; - x = strtod(s, NULL); - bp->x[0] = x; + if (dim > 0) + bp->x[i++] = float8in_internal(s, &endptr, "cube", str); while ((s = strchr(s, ',')) != NULL) { - s++; i++; - x = strtod(s, NULL); - bp->x[i] = x; + s++; + bp->x[i++] = float8in_internal(s, &endptr, "cube", str); } + Assert(i == dim); - return(bp); + return bp; } #include "cubescan.c" diff --git a/contrib/cube/cubescan.l b/contrib/cube/cubescan.l index 4408e28387..dada917820 100644 --- a/contrib/cube/cubescan.l +++ b/contrib/cube/cubescan.l @@ -38,36 +38,41 @@ n [0-9]+ integer [+-]?{n} real [+-]?({n}\.{n}?|\.{n}) float ({integer}|{real})([eE]{integer})? +infinity [+-]?[iI][nN][fF]([iI][nN][iI][tT][yY])? +NaN [nN][aA][nN] %% {float} yylval = yytext; return CUBEFLOAT; +{infinity} yylval = yytext; return CUBEFLOAT; +{NaN} yylval = yytext; return CUBEFLOAT; \[ yylval = "("; return O_BRACKET; \] yylval = ")"; return C_BRACKET; \( yylval = "("; return O_PAREN; \) yylval = ")"; return C_PAREN; -\, yylval = ")"; return COMMA; +\, yylval = ","; return COMMA; [ \t\n\r\f]+ /* discard spaces */ . return yytext[0]; /* alert parser of the garbage */ %% +/* result is not used, but Bison expects this signature */ void yyerror(NDBOX **result, const char *message) { if (*yytext == YY_END_OF_BUFFER_CHAR) { ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("bad cube representation"), + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for cube"), /* translator: %s is typically "syntax error" */ errdetail("%s at end of input", message))); } else { ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("bad cube representation"), + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for cube"), /* translator: first %s is typically "syntax error" */ errdetail("%s at or near \"%s\"", message, yytext))); } diff --git a/contrib/cube/expected/cube.out b/contrib/cube/expected/cube.out index e9e2c0f15b..095491fa0c 100644 --- a/contrib/cube/expected/cube.out +++ b/contrib/cube/expected/cube.out @@ -126,16 +126,34 @@ SELECT '-1.0e-7'::cube AS cube; (-1e-07) (1 row) -SELECT '1e-700'::cube AS cube; - cube ------- - (0) +SELECT '1e-300'::cube AS cube; + cube +---------- + (1e-300) (1 row) -SELECT '-1e-700'::cube AS cube; - cube ------- - (0) +SELECT '-1e-300'::cube AS cube; + cube +----------- + (-1e-300) +(1 row) + +SELECT 'infinity'::cube AS cube; + cube +------------ + (Infinity) +(1 row) + +SELECT '-infinity'::cube AS cube; + cube +------------- + (-Infinity) +(1 row) + +SELECT 'NaN'::cube AS cube; + cube +------- + (NaN) (1 row) SELECT '1234567890123456'::cube AS cube; @@ -175,6 +193,12 @@ SELECT '-.1234567890123456'::cube AS cube; (1 row) -- simple lists (points) +SELECT '()'::cube AS cube; + cube +------ + () +(1 row) + SELECT '1,2'::cube AS cube; cube -------- @@ -200,6 +224,12 @@ SELECT '(1,2,3,4,5)'::cube AS cube; (1 row) -- double lists (cubes) +SELECT '(),()'::cube AS cube; + cube +------ + () +(1 row) + SELECT '(0),(0)'::cube AS cube; cube ------ @@ -250,146 +280,145 @@ SELECT '[(0,0,0,0),(1,0,0,0)]'::cube AS cube; -- invalid input: parse errors SELECT ''::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT ''::cube AS cube; ^ DETAIL: syntax error at end of input SELECT 'ABC'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT 'ABC'::cube AS cube; ^ DETAIL: syntax error at or near "A" -SELECT '()'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '()'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" SELECT '[]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[]'::cube AS cube; ^ DETAIL: syntax error at or near "]" SELECT '[()]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[()]'::cube AS cube; ^ -DETAIL: syntax error at or near ")" +DETAIL: syntax error at or near "]" SELECT '[(1)]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1)]'::cube AS cube; ^ DETAIL: syntax error at or near "]" SELECT '[(1),]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1),]'::cube AS cube; ^ DETAIL: syntax error at or near "]" SELECT '[(1),2]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1),2]'::cube AS cube; ^ DETAIL: syntax error at or near "2" SELECT '[(1),(2),(3)]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1),(2),(3)]'::cube AS cube; ^ DETAIL: syntax error at or near "," SELECT '1,'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,'::cube AS cube; ^ DETAIL: syntax error at end of input SELECT '1,2,'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,2,'::cube AS cube; ^ DETAIL: syntax error at end of input SELECT '1,,2'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,,2'::cube AS cube; ^ DETAIL: syntax error at or near "," SELECT '(1,)'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,)'::cube AS cube; ^ DETAIL: syntax error at or near ")" SELECT '(1,2,)'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2,)'::cube AS cube; ^ DETAIL: syntax error at or near ")" SELECT '(1,,2)'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,,2)'::cube AS cube; ^ DETAIL: syntax error at or near "," -- invalid input: semantic errors and trailing garbage SELECT '[(1),(2)],'::cube AS cube; -- 0 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1),(2)],'::cube AS cube; ^ DETAIL: syntax error at or near "," SELECT '[(1,2,3),(2,3)]'::cube AS cube; -- 1 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1,2,3),(2,3)]'::cube AS cube; ^ DETAIL: Different point dimensions in (1,2,3) and (2,3). SELECT '[(1,2),(1,2,3)]'::cube AS cube; -- 1 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1,2),(1,2,3)]'::cube AS cube; ^ DETAIL: Different point dimensions in (1,2) and (1,2,3). SELECT '(1),(2),'::cube AS cube; -- 2 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1),(2),'::cube AS cube; ^ DETAIL: syntax error at or near "," SELECT '(1,2,3),(2,3)'::cube AS cube; -- 3 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2,3),(2,3)'::cube AS cube; ^ DETAIL: Different point dimensions in (1,2,3) and (2,3). SELECT '(1,2),(1,2,3)'::cube AS cube; -- 3 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2),(1,2,3)'::cube AS cube; ^ DETAIL: Different point dimensions in (1,2) and (1,2,3). SELECT '(1,2,3)ab'::cube AS cube; -- 4 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2,3)ab'::cube AS cube; ^ DETAIL: syntax error at or near "a" SELECT '(1,2,3)a'::cube AS cube; -- 5 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2,3)a'::cube AS cube; ^ DETAIL: syntax error at or near "a" SELECT '(1,2)('::cube AS cube; -- 5 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2)('::cube AS cube; ^ DETAIL: syntax error at or near "(" SELECT '1,2ab'::cube AS cube; -- 6 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,2ab'::cube AS cube; ^ DETAIL: syntax error at or near "a" SELECT '1 e7'::cube AS cube; -- 6 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1 e7'::cube AS cube; ^ DETAIL: syntax error at or near "e" SELECT '1,2a'::cube AS cube; -- 7 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,2a'::cube AS cube; ^ DETAIL: syntax error at or near "a" SELECT '1..2'::cube AS cube; -- 7 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1..2'::cube AS cube; ^ DETAIL: syntax error at or near ".2" +SELECT '-1e-700'::cube AS cube; -- out of range +ERROR: "-1e-700" is out of range for type double precision +LINE 1: SELECT '-1e-700'::cube AS cube; + ^ -- -- Testing building cubes from float8 values -- @@ -556,12 +585,12 @@ SELECT cube(cube(1,2), 42, 24); -- cube_c_f8_f8 -- Testing limit of CUBE_MAX_DIM dimensions check in cube_in. -- select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)'::cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0... ^ DETAIL: A cube cannot have more than 100 dimensions. select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0),(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)'::cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0... ^ DETAIL: A cube cannot have more than 100 dimensions. diff --git a/contrib/cube/expected/cube_1.out b/contrib/cube/expected/cube_1.out deleted file mode 100644 index c40fabcd46..0000000000 --- a/contrib/cube/expected/cube_1.out +++ /dev/null @@ -1,1710 +0,0 @@ --- --- Test cube datatype --- -CREATE EXTENSION cube; --- --- testing the input and output functions --- --- Any number (a one-dimensional point) -SELECT '1'::cube AS cube; - cube ------- - (1) -(1 row) - -SELECT '-1'::cube AS cube; - cube ------- - (-1) -(1 row) - -SELECT '1.'::cube AS cube; - cube ------- - (1) -(1 row) - -SELECT '-1.'::cube AS cube; - cube ------- - (-1) -(1 row) - -SELECT '.1'::cube AS cube; - cube -------- - (0.1) -(1 row) - -SELECT '-.1'::cube AS cube; - cube --------- - (-0.1) -(1 row) - -SELECT '1.0'::cube AS cube; - cube ------- - (1) -(1 row) - -SELECT '-1.0'::cube AS cube; - cube ------- - (-1) -(1 row) - -SELECT '1e27'::cube AS cube; - cube ---------- - (1e+27) -(1 row) - -SELECT '-1e27'::cube AS cube; - cube ----------- - (-1e+27) -(1 row) - -SELECT '1.0e27'::cube AS cube; - cube ---------- - (1e+27) -(1 row) - -SELECT '-1.0e27'::cube AS cube; - cube ----------- - (-1e+27) -(1 row) - -SELECT '1e+27'::cube AS cube; - cube ---------- - (1e+27) -(1 row) - -SELECT '-1e+27'::cube AS cube; - cube ----------- - (-1e+27) -(1 row) - -SELECT '1.0e+27'::cube AS cube; - cube ---------- - (1e+27) -(1 row) - -SELECT '-1.0e+27'::cube AS cube; - cube ----------- - (-1e+27) -(1 row) - -SELECT '1e-7'::cube AS cube; - cube ---------- - (1e-07) -(1 row) - -SELECT '-1e-7'::cube AS cube; - cube ----------- - (-1e-07) -(1 row) - -SELECT '1.0e-7'::cube AS cube; - cube ---------- - (1e-07) -(1 row) - -SELECT '-1.0e-7'::cube AS cube; - cube ----------- - (-1e-07) -(1 row) - -SELECT '1e-700'::cube AS cube; - cube ------- - (0) -(1 row) - -SELECT '-1e-700'::cube AS cube; - cube ------- - (-0) -(1 row) - -SELECT '1234567890123456'::cube AS cube; - cube ------------------------- - (1.23456789012346e+15) -(1 row) - -SELECT '+1234567890123456'::cube AS cube; - cube ------------------------- - (1.23456789012346e+15) -(1 row) - -SELECT '-1234567890123456'::cube AS cube; - cube -------------------------- - (-1.23456789012346e+15) -(1 row) - -SELECT '.1234567890123456'::cube AS cube; - cube ---------------------- - (0.123456789012346) -(1 row) - -SELECT '+.1234567890123456'::cube AS cube; - cube ---------------------- - (0.123456789012346) -(1 row) - -SELECT '-.1234567890123456'::cube AS cube; - cube ----------------------- - (-0.123456789012346) -(1 row) - --- simple lists (points) -SELECT '1,2'::cube AS cube; - cube --------- - (1, 2) -(1 row) - -SELECT '(1,2)'::cube AS cube; - cube --------- - (1, 2) -(1 row) - -SELECT '1,2,3,4,5'::cube AS cube; - cube ------------------ - (1, 2, 3, 4, 5) -(1 row) - -SELECT '(1,2,3,4,5)'::cube AS cube; - cube ------------------ - (1, 2, 3, 4, 5) -(1 row) - --- double lists (cubes) -SELECT '(0),(0)'::cube AS cube; - cube ------- - (0) -(1 row) - -SELECT '(0),(1)'::cube AS cube; - cube ---------- - (0),(1) -(1 row) - -SELECT '[(0),(0)]'::cube AS cube; - cube ------- - (0) -(1 row) - -SELECT '[(0),(1)]'::cube AS cube; - cube ---------- - (0),(1) -(1 row) - -SELECT '(0,0,0,0),(0,0,0,0)'::cube AS cube; - cube --------------- - (0, 0, 0, 0) -(1 row) - -SELECT '(0,0,0,0),(1,0,0,0)'::cube AS cube; - cube ---------------------------- - (0, 0, 0, 0),(1, 0, 0, 0) -(1 row) - -SELECT '[(0,0,0,0),(0,0,0,0)]'::cube AS cube; - cube --------------- - (0, 0, 0, 0) -(1 row) - -SELECT '[(0,0,0,0),(1,0,0,0)]'::cube AS cube; - cube ---------------------------- - (0, 0, 0, 0),(1, 0, 0, 0) -(1 row) - --- invalid input: parse errors -SELECT ''::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT ''::cube AS cube; - ^ -DETAIL: syntax error at end of input -SELECT 'ABC'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT 'ABC'::cube AS cube; - ^ -DETAIL: syntax error at or near "A" -SELECT '()'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '()'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" -SELECT '[]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[]'::cube AS cube; - ^ -DETAIL: syntax error at or near "]" -SELECT '[()]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[()]'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" -SELECT '[(1)]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[(1)]'::cube AS cube; - ^ -DETAIL: syntax error at or near "]" -SELECT '[(1),]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[(1),]'::cube AS cube; - ^ -DETAIL: syntax error at or near "]" -SELECT '[(1),2]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[(1),2]'::cube AS cube; - ^ -DETAIL: syntax error at or near "2" -SELECT '[(1),(2),(3)]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[(1),(2),(3)]'::cube AS cube; - ^ -DETAIL: syntax error at or near "," -SELECT '1,'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '1,'::cube AS cube; - ^ -DETAIL: syntax error at end of input -SELECT '1,2,'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '1,2,'::cube AS cube; - ^ -DETAIL: syntax error at end of input -SELECT '1,,2'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '1,,2'::cube AS cube; - ^ -DETAIL: syntax error at or near "," -SELECT '(1,)'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '(1,)'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" -SELECT '(1,2,)'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '(1,2,)'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" -SELECT '(1,,2)'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '(1,,2)'::cube AS cube; - ^ -DETAIL: syntax error at or near "," --- invalid input: semantic errors and trailing garbage -SELECT '[(1),(2)],'::cube AS cube; -- 0 -ERROR: bad cube representation -LINE 1: SELECT '[(1),(2)],'::cube AS cube; - ^ -DETAIL: syntax error at or near "," -SELECT '[(1,2,3),(2,3)]'::cube AS cube; -- 1 -ERROR: bad cube representation -LINE 1: SELECT '[(1,2,3),(2,3)]'::cube AS cube; - ^ -DETAIL: Different point dimensions in (1,2,3) and (2,3). -SELECT '[(1,2),(1,2,3)]'::cube AS cube; -- 1 -ERROR: bad cube representation -LINE 1: SELECT '[(1,2),(1,2,3)]'::cube AS cube; - ^ -DETAIL: Different point dimensions in (1,2) and (1,2,3). -SELECT '(1),(2),'::cube AS cube; -- 2 -ERROR: bad cube representation -LINE 1: SELECT '(1),(2),'::cube AS cube; - ^ -DETAIL: syntax error at or near "," -SELECT '(1,2,3),(2,3)'::cube AS cube; -- 3 -ERROR: bad cube representation -LINE 1: SELECT '(1,2,3),(2,3)'::cube AS cube; - ^ -DETAIL: Different point dimensions in (1,2,3) and (2,3). -SELECT '(1,2),(1,2,3)'::cube AS cube; -- 3 -ERROR: bad cube representation -LINE 1: SELECT '(1,2),(1,2,3)'::cube AS cube; - ^ -DETAIL: Different point dimensions in (1,2) and (1,2,3). -SELECT '(1,2,3)ab'::cube AS cube; -- 4 -ERROR: bad cube representation -LINE 1: SELECT '(1,2,3)ab'::cube AS cube; - ^ -DETAIL: syntax error at or near "a" -SELECT '(1,2,3)a'::cube AS cube; -- 5 -ERROR: bad cube representation -LINE 1: SELECT '(1,2,3)a'::cube AS cube; - ^ -DETAIL: syntax error at or near "a" -SELECT '(1,2)('::cube AS cube; -- 5 -ERROR: bad cube representation -LINE 1: SELECT '(1,2)('::cube AS cube; - ^ -DETAIL: syntax error at or near "(" -SELECT '1,2ab'::cube AS cube; -- 6 -ERROR: bad cube representation -LINE 1: SELECT '1,2ab'::cube AS cube; - ^ -DETAIL: syntax error at or near "a" -SELECT '1 e7'::cube AS cube; -- 6 -ERROR: bad cube representation -LINE 1: SELECT '1 e7'::cube AS cube; - ^ -DETAIL: syntax error at or near "e" -SELECT '1,2a'::cube AS cube; -- 7 -ERROR: bad cube representation -LINE 1: SELECT '1,2a'::cube AS cube; - ^ -DETAIL: syntax error at or near "a" -SELECT '1..2'::cube AS cube; -- 7 -ERROR: bad cube representation -LINE 1: SELECT '1..2'::cube AS cube; - ^ -DETAIL: syntax error at or near ".2" --- --- Testing building cubes from float8 values --- -SELECT cube(0::float8); - cube ------- - (0) -(1 row) - -SELECT cube(1::float8); - cube ------- - (1) -(1 row) - -SELECT cube(1,2); - cube ---------- - (1),(2) -(1 row) - -SELECT cube(cube(1,2),3); - cube ---------------- - (1, 3),(2, 3) -(1 row) - -SELECT cube(cube(1,2),3,4); - cube ---------------- - (1, 3),(2, 4) -(1 row) - -SELECT cube(cube(cube(1,2),3,4),5); - cube ---------------------- - (1, 3, 5),(2, 4, 5) -(1 row) - -SELECT cube(cube(cube(1,2),3,4),5,6); - cube ---------------------- - (1, 3, 5),(2, 4, 6) -(1 row) - --- --- Test that the text -> cube cast was installed. --- -SELECT '(0)'::text::cube; - cube ------- - (0) -(1 row) - --- --- Test the float[] -> cube cast --- -SELECT cube('{0,1,2}'::float[], '{3,4,5}'::float[]); - cube ---------------------- - (0, 1, 2),(3, 4, 5) -(1 row) - -SELECT cube('{0,1,2}'::float[], '{3}'::float[]); -ERROR: UR and LL arrays must be of same length -SELECT cube(NULL::float[], '{3}'::float[]); - cube ------- - -(1 row) - -SELECT cube('{0,1,2}'::float[]); - cube ------------ - (0, 1, 2) -(1 row) - -SELECT cube_subset(cube('(1,3,5),(6,7,8)'), ARRAY[3,2,1,1]); - cube_subset ---------------------------- - (5, 3, 1, 1),(8, 7, 6, 6) -(1 row) - -SELECT cube_subset(cube('(1,3,5),(1,3,5)'), ARRAY[3,2,1,1]); - cube_subset --------------- - (5, 3, 1, 1) -(1 row) - -SELECT cube_subset(cube('(1,3,5),(6,7,8)'), ARRAY[4,0]); -ERROR: Index out of bounds -SELECT cube_subset(cube('(6,7,8),(6,7,8)'), ARRAY[4,0]); -ERROR: Index out of bounds --- --- Test point processing --- -SELECT cube('(1,2),(1,2)'); -- cube_in - cube --------- - (1, 2) -(1 row) - -SELECT cube('{0,1,2}'::float[], '{0,1,2}'::float[]); -- cube_a_f8_f8 - cube ------------ - (0, 1, 2) -(1 row) - -SELECT cube('{5,6,7,8}'::float[]); -- cube_a_f8 - cube --------------- - (5, 6, 7, 8) -(1 row) - -SELECT cube(1.37); -- cube_f8 - cube --------- - (1.37) -(1 row) - -SELECT cube(1.37, 1.37); -- cube_f8_f8 - cube --------- - (1.37) -(1 row) - -SELECT cube(cube(1,1), 42); -- cube_c_f8 - cube ---------- - (1, 42) -(1 row) - -SELECT cube(cube(1,2), 42); -- cube_c_f8 - cube ------------------ - (1, 42),(2, 42) -(1 row) - -SELECT cube(cube(1,1), 42, 42); -- cube_c_f8_f8 - cube ---------- - (1, 42) -(1 row) - -SELECT cube(cube(1,1), 42, 24); -- cube_c_f8_f8 - cube ------------------ - (1, 42),(1, 24) -(1 row) - -SELECT cube(cube(1,2), 42, 42); -- cube_c_f8_f8 - cube ------------------ - (1, 42),(2, 42) -(1 row) - -SELECT cube(cube(1,2), 42, 24); -- cube_c_f8_f8 - cube ------------------ - (1, 42),(2, 24) -(1 row) - --- --- Testing limit of CUBE_MAX_DIM dimensions check in cube_in. --- -select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)'::cube; -ERROR: bad cube representation -LINE 1: select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0... - ^ -DETAIL: A cube cannot have more than 100 dimensions. -select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0),(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)'::cube; -ERROR: bad cube representation -LINE 1: select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0... - ^ -DETAIL: A cube cannot have more than 100 dimensions. --- --- testing the operators --- --- equality/inequality: --- -SELECT '24, 33.20'::cube = '24, 33.20'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '24, 33.20'::cube != '24, 33.20'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '24, 33.20'::cube = '24, 33.21'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '24, 33.20'::cube != '24, 33.21'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0),(3,1)'::cube = '(2,0,0,0,0),(3,1,0,0,0)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0),(3,1)'::cube = '(2,0,0,0,0),(3,1,0,0,1)'::cube AS bool; - bool ------- - f -(1 row) - --- "lower than" / "greater than" --- (these operators are not useful for anything but ordering) --- -SELECT '1'::cube > '2'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '1'::cube < '2'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '1,1'::cube > '1,2'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '1,1'::cube < '1,2'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0),(3,1)'::cube > '(2,0,0,0,0),(3,1,0,0,1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0),(3,1)'::cube < '(2,0,0,0,0),(3,1,0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0),(3,1)'::cube > '(2,0,0,0,1),(3,1,0,0,0)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0),(3,1)'::cube < '(2,0,0,0,1),(3,1,0,0,0)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0),(3,1)'::cube > '(2,0,0,0,0),(3,1,0,0,0)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0),(3,1)'::cube < '(2,0,0,0,0),(3,1,0,0,0)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0,0,0,0),(3,1,0,0,1)'::cube > '(2,0),(3,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0,0,0,0),(3,1,0,0,1)'::cube < '(2,0),(3,1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0,0,0,1),(3,1,0,0,0)'::cube > '(2,0),(3,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0,0,0,1),(3,1,0,0,0)'::cube < '(2,0),(3,1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0,0,0,0),(3,1,0,0,0)'::cube > '(2,0),(3,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0,0,0,0),(3,1,0,0,0)'::cube < '(2,0),(3,1)'::cube AS bool; - bool ------- - f -(1 row) - --- "overlap" --- -SELECT '1'::cube && '1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '1'::cube && '2'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '1,1,1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '[(1,1,1),(2,2,2)]'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '[(1,1),(2,2)]'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '[(2,1,1),(2,2,2)]'::cube AS bool; - bool ------- - f -(1 row) - --- "contained in" (the left operand is the cube entirely enclosed by --- the right operand): --- -SELECT '0'::cube <@ '0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,0'::cube <@ '0,0,0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0'::cube <@ '0,0,1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,0'::cube <@ '0,0,1'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '1,0,0'::cube <@ '0,0,1'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(1,0,0),(0,0,1)'::cube <@ '(1,0,0),(0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(1,0,0),(0,0,1)'::cube <@ '(-1,-1,-1),(1,1,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(1,0,0),(0,0,1)'::cube <@ '(-1,-1,-1,-1),(1,1,1,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '1'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '-1'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube <@ '(-1,-1),(1,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-2),(1)'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(-2),(1)'::cube <@ '(-1,-1),(1,1)'::cube AS bool; - bool ------- - f -(1 row) - --- "contains" (the left operand is the cube that entirely encloses the --- right operand) --- -SELECT '0'::cube @> '0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,0'::cube @> '0,0,0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,1'::cube @> '0,0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,1'::cube @> '0,0,0'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '0,0,1'::cube @> '1,0,0'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(1,0,0),(0,0,1)'::cube @> '(1,0,0),(0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1,-1,-1),(1,1,1)'::cube @> '(1,0,0),(0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1,-1,-1,-1),(1,1,1,1)'::cube @> '(1,0,0),(0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '-1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1,-1),(1,1)'::cube @> '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '(-2),(1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(-1,-1),(1,1)'::cube @> '(-2),(1)'::cube AS bool; - bool ------- - f -(1 row) - --- Test of distance function --- -SELECT cube_distance('(0)'::cube,'(2,2,2,2)'::cube); - cube_distance ---------------- - 4 -(1 row) - -SELECT cube_distance('(0)'::cube,'(.3,.4)'::cube); - cube_distance ---------------- - 0.5 -(1 row) - -SELECT cube_distance('(2,3,4)'::cube,'(2,3,4)'::cube); - cube_distance ---------------- - 0 -(1 row) - -SELECT cube_distance('(42,42,42,42)'::cube,'(137,137,137,137)'::cube); - cube_distance ---------------- - 190 -(1 row) - -SELECT cube_distance('(42,42,42)'::cube,'(137,137)'::cube); - cube_distance ------------------- - 140.762210837994 -(1 row) - --- Test of cube function (text to cube) --- -SELECT cube('(1,1.2)'::text); - cube ----------- - (1, 1.2) -(1 row) - -SELECT cube(NULL); - cube ------- - -(1 row) - --- Test of cube_dim function (dimensions stored in cube) --- -SELECT cube_dim('(0)'::cube); - cube_dim ----------- - 1 -(1 row) - -SELECT cube_dim('(0,0)'::cube); - cube_dim ----------- - 2 -(1 row) - -SELECT cube_dim('(0,0,0)'::cube); - cube_dim ----------- - 3 -(1 row) - -SELECT cube_dim('(42,42,42),(42,42,42)'::cube); - cube_dim ----------- - 3 -(1 row) - -SELECT cube_dim('(4,8,15,16,23),(4,8,15,16,23)'::cube); - cube_dim ----------- - 5 -(1 row) - --- Test of cube_ll_coord function (retrieves LL coodinate values) --- -SELECT cube_ll_coord('(-1,1),(2,-2)'::cube, 1); - cube_ll_coord ---------------- - -1 -(1 row) - -SELECT cube_ll_coord('(-1,1),(2,-2)'::cube, 2); - cube_ll_coord ---------------- - -2 -(1 row) - -SELECT cube_ll_coord('(-1,1),(2,-2)'::cube, 3); - cube_ll_coord ---------------- - 0 -(1 row) - -SELECT cube_ll_coord('(1,2),(1,2)'::cube, 1); - cube_ll_coord ---------------- - 1 -(1 row) - -SELECT cube_ll_coord('(1,2),(1,2)'::cube, 2); - cube_ll_coord ---------------- - 2 -(1 row) - -SELECT cube_ll_coord('(1,2),(1,2)'::cube, 3); - cube_ll_coord ---------------- - 0 -(1 row) - -SELECT cube_ll_coord('(42,137)'::cube, 1); - cube_ll_coord ---------------- - 42 -(1 row) - -SELECT cube_ll_coord('(42,137)'::cube, 2); - cube_ll_coord ---------------- - 137 -(1 row) - -SELECT cube_ll_coord('(42,137)'::cube, 3); - cube_ll_coord ---------------- - 0 -(1 row) - --- Test of cube_ur_coord function (retrieves UR coodinate values) --- -SELECT cube_ur_coord('(-1,1),(2,-2)'::cube, 1); - cube_ur_coord ---------------- - 2 -(1 row) - -SELECT cube_ur_coord('(-1,1),(2,-2)'::cube, 2); - cube_ur_coord ---------------- - 1 -(1 row) - -SELECT cube_ur_coord('(-1,1),(2,-2)'::cube, 3); - cube_ur_coord ---------------- - 0 -(1 row) - -SELECT cube_ur_coord('(1,2),(1,2)'::cube, 1); - cube_ur_coord ---------------- - 1 -(1 row) - -SELECT cube_ur_coord('(1,2),(1,2)'::cube, 2); - cube_ur_coord ---------------- - 2 -(1 row) - -SELECT cube_ur_coord('(1,2),(1,2)'::cube, 3); - cube_ur_coord ---------------- - 0 -(1 row) - -SELECT cube_ur_coord('(42,137)'::cube, 1); - cube_ur_coord ---------------- - 42 -(1 row) - -SELECT cube_ur_coord('(42,137)'::cube, 2); - cube_ur_coord ---------------- - 137 -(1 row) - -SELECT cube_ur_coord('(42,137)'::cube, 3); - cube_ur_coord ---------------- - 0 -(1 row) - --- Test of cube_is_point --- -SELECT cube_is_point('(0)'::cube); - cube_is_point ---------------- - t -(1 row) - -SELECT cube_is_point('(0,1,2)'::cube); - cube_is_point ---------------- - t -(1 row) - -SELECT cube_is_point('(0,1,2),(0,1,2)'::cube); - cube_is_point ---------------- - t -(1 row) - -SELECT cube_is_point('(0,1,2),(-1,1,2)'::cube); - cube_is_point ---------------- - f -(1 row) - -SELECT cube_is_point('(0,1,2),(0,-1,2)'::cube); - cube_is_point ---------------- - f -(1 row) - -SELECT cube_is_point('(0,1,2),(0,1,-2)'::cube); - cube_is_point ---------------- - f -(1 row) - --- Test of cube_enlarge (enlarging and shrinking cubes) --- -SELECT cube_enlarge('(0)'::cube, 0, 0); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 0, 1); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 0, 2); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(2),(-2)'::cube, 0, 4); - cube_enlarge --------------- - (-2),(2) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 1, 0); - cube_enlarge --------------- - (-1),(1) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 1, 1); - cube_enlarge --------------- - (-1),(1) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 1, 2); - cube_enlarge ------------------ - (-1, -1),(1, 1) -(1 row) - -SELECT cube_enlarge('(2),(-2)'::cube, 1, 4); - cube_enlarge -------------------------------- - (-3, -1, -1, -1),(3, 1, 1, 1) -(1 row) - -SELECT cube_enlarge('(0)'::cube, -1, 0); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(0)'::cube, -1, 1); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(0)'::cube, -1, 2); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(2),(-2)'::cube, -1, 4); - cube_enlarge --------------- - (-1),(1) -(1 row) - -SELECT cube_enlarge('(0,0,0)'::cube, 1, 0); - cube_enlarge ------------------------- - (-1, -1, -1),(1, 1, 1) -(1 row) - -SELECT cube_enlarge('(0,0,0)'::cube, 1, 2); - cube_enlarge ------------------------- - (-1, -1, -1),(1, 1, 1) -(1 row) - -SELECT cube_enlarge('(2,-2),(-3,7)'::cube, 1, 2); - cube_enlarge ------------------ - (-4, -3),(3, 8) -(1 row) - -SELECT cube_enlarge('(2,-2),(-3,7)'::cube, 3, 2); - cube_enlarge ------------------- - (-6, -5),(5, 10) -(1 row) - -SELECT cube_enlarge('(2,-2),(-3,7)'::cube, -1, 2); - cube_enlarge ------------------ - (-2, -1),(1, 6) -(1 row) - -SELECT cube_enlarge('(2,-2),(-3,7)'::cube, -3, 2); - cube_enlarge ---------------------- - (-0.5, 1),(-0.5, 4) -(1 row) - -SELECT cube_enlarge('(42,-23,-23),(42,23,23)'::cube, -23, 5); - cube_enlarge --------------- - (42, 0, 0) -(1 row) - -SELECT cube_enlarge('(42,-23,-23),(42,23,23)'::cube, -24, 5); - cube_enlarge --------------- - (42, 0, 0) -(1 row) - --- Test of cube_union (MBR for two cubes) --- -SELECT cube_union('(1,2),(3,4)'::cube, '(5,6,7),(8,9,10)'::cube); - cube_union ----------------------- - (1, 2, 0),(8, 9, 10) -(1 row) - -SELECT cube_union('(1,2)'::cube, '(4,2,0,0)'::cube); - cube_union ---------------------------- - (1, 2, 0, 0),(4, 2, 0, 0) -(1 row) - -SELECT cube_union('(1,2),(1,2)'::cube, '(4,2),(4,2)'::cube); - cube_union ---------------- - (1, 2),(4, 2) -(1 row) - -SELECT cube_union('(1,2),(1,2)'::cube, '(1,2),(1,2)'::cube); - cube_union ------------- - (1, 2) -(1 row) - -SELECT cube_union('(1,2),(1,2)'::cube, '(1,2,0),(1,2,0)'::cube); - cube_union ------------- - (1, 2, 0) -(1 row) - --- Test of cube_inter --- -SELECT cube_inter('(1,2),(10,11)'::cube, '(3,4), (16,15)'::cube); -- intersects - cube_inter ------------------ - (3, 4),(10, 11) -(1 row) - -SELECT cube_inter('(1,2),(10,11)'::cube, '(3,4), (6,5)'::cube); -- includes - cube_inter ---------------- - (3, 4),(6, 5) -(1 row) - -SELECT cube_inter('(1,2),(10,11)'::cube, '(13,14), (16,15)'::cube); -- no intersection - cube_inter -------------------- - (13, 14),(10, 11) -(1 row) - -SELECT cube_inter('(1,2),(10,11)'::cube, '(3,14), (16,15)'::cube); -- no intersection, but one dimension intersects - cube_inter ------------------- - (3, 14),(10, 11) -(1 row) - -SELECT cube_inter('(1,2),(10,11)'::cube, '(10,11), (16,15)'::cube); -- point intersection - cube_inter ------------- - (10, 11) -(1 row) - -SELECT cube_inter('(1,2,3)'::cube, '(1,2,3)'::cube); -- point args - cube_inter ------------- - (1, 2, 3) -(1 row) - -SELECT cube_inter('(1,2,3)'::cube, '(5,6,3)'::cube); -- point args - cube_inter ---------------------- - (5, 6, 3),(1, 2, 3) -(1 row) - --- Test of cube_size --- -SELECT cube_size('(4,8),(15,16)'::cube); - cube_size ------------ - 88 -(1 row) - -SELECT cube_size('(42,137)'::cube); - cube_size ------------ - 0 -(1 row) - --- Test of distances --- -SELECT cube_distance('(1,1)'::cube, '(4,5)'::cube); - cube_distance ---------------- - 5 -(1 row) - -SELECT '(1,1)'::cube <-> '(4,5)'::cube as d_e; - d_e ------ - 5 -(1 row) - -SELECT distance_chebyshev('(1,1)'::cube, '(4,5)'::cube); - distance_chebyshev --------------------- - 4 -(1 row) - -SELECT '(1,1)'::cube <=> '(4,5)'::cube as d_c; - d_c ------ - 4 -(1 row) - -SELECT distance_taxicab('(1,1)'::cube, '(4,5)'::cube); - distance_taxicab ------------------- - 7 -(1 row) - -SELECT '(1,1)'::cube <#> '(4,5)'::cube as d_t; - d_t ------ - 7 -(1 row) - --- zero for overlapping -SELECT cube_distance('(2,2),(10,10)'::cube, '(0,0),(5,5)'::cube); - cube_distance ---------------- - 0 -(1 row) - -SELECT distance_chebyshev('(2,2),(10,10)'::cube, '(0,0),(5,5)'::cube); - distance_chebyshev --------------------- - 0 -(1 row) - -SELECT distance_taxicab('(2,2),(10,10)'::cube, '(0,0),(5,5)'::cube); - distance_taxicab ------------------- - 0 -(1 row) - --- coordinate access -SELECT cube(array[10,20,30], array[40,50,60])->1; - ?column? ----------- - 10 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])->1; - ?column? ----------- - 40 -(1 row) - -SELECT cube(array[10,20,30], array[40,50,60])->6; - ?column? ----------- - 60 -(1 row) - -SELECT cube(array[10,20,30], array[40,50,60])->0; -ERROR: cube index 0 is out of bounds -SELECT cube(array[10,20,30], array[40,50,60])->7; -ERROR: cube index 7 is out of bounds -SELECT cube(array[10,20,30], array[40,50,60])->-1; -ERROR: cube index -1 is out of bounds -SELECT cube(array[10,20,30], array[40,50,60])->-6; -ERROR: cube index -6 is out of bounds -SELECT cube(array[10,20,30])->3; - ?column? ----------- - 30 -(1 row) - -SELECT cube(array[10,20,30])->6; - ?column? ----------- - 30 -(1 row) - -SELECT cube(array[10,20,30])->-6; -ERROR: cube index -6 is out of bounds --- "normalized" coordinate access -SELECT cube(array[10,20,30], array[40,50,60])~>1; - ?column? ----------- - 10 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>1; - ?column? ----------- - 10 -(1 row) - -SELECT cube(array[10,20,30], array[40,50,60])~>2; - ?column? ----------- - 20 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>2; - ?column? ----------- - 20 -(1 row) - -SELECT cube(array[10,20,30], array[40,50,60])~>3; - ?column? ----------- - 30 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>3; - ?column? ----------- - 30 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>0; -ERROR: cube index 0 is out of bounds -SELECT cube(array[40,50,60], array[10,20,30])~>4; - ?column? ----------- - 40 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>(-1); -ERROR: cube index -1 is out of bounds --- Load some example data and build the index --- -CREATE TABLE test_cube (c cube); -\copy test_cube from 'data/test_cube.data' -CREATE INDEX test_cube_ix ON test_cube USING gist (c); -SELECT * FROM test_cube WHERE c && '(3000,1000),(0,0)' ORDER BY c; - c --------------------------- - (337, 455),(240, 359) - (759, 187),(662, 163) - (1444, 403),(1346, 344) - (1594, 1043),(1517, 971) - (2424, 160),(2424, 81) -(5 rows) - --- Test sorting -SELECT * FROM test_cube WHERE c && '(3000,1000),(0,0)' GROUP BY c ORDER BY c; - c --------------------------- - (337, 455),(240, 359) - (759, 187),(662, 163) - (1444, 403),(1346, 344) - (1594, 1043),(1517, 971) - (2424, 160),(2424, 81) -(5 rows) - --- kNN with index -SELECT *, c <-> '(100, 100),(500, 500)'::cube as dist FROM test_cube ORDER BY c <-> '(100, 100),(500, 500)'::cube LIMIT 5; - c | dist --------------------------+------------------ - (337, 455),(240, 359) | 0 - (759, 187),(662, 163) | 162 - (948, 1201),(907, 1156) | 772.000647668122 - (1444, 403),(1346, 344) | 846 - (369, 1457),(278, 1409) | 909 -(5 rows) - -SELECT *, c <=> '(100, 100),(500, 500)'::cube as dist FROM test_cube ORDER BY c <=> '(100, 100),(500, 500)'::cube LIMIT 5; - c | dist --------------------------+------ - (337, 455),(240, 359) | 0 - (759, 187),(662, 163) | 162 - (948, 1201),(907, 1156) | 656 - (1444, 403),(1346, 344) | 846 - (369, 1457),(278, 1409) | 909 -(5 rows) - -SELECT *, c <#> '(100, 100),(500, 500)'::cube as dist FROM test_cube ORDER BY c <#> '(100, 100),(500, 500)'::cube LIMIT 5; - c | dist --------------------------+------ - (337, 455),(240, 359) | 0 - (759, 187),(662, 163) | 162 - (1444, 403),(1346, 344) | 846 - (369, 1457),(278, 1409) | 909 - (948, 1201),(907, 1156) | 1063 -(5 rows) - --- kNN-based sorting -SELECT * FROM test_cube ORDER BY c~>1 LIMIT 15; -- ascending by 1st coordinate of lower left corner - c ---------------------------- - (54, 38679),(3, 38602) - (83, 10271),(15, 10265) - (122, 46832),(64, 46762) - (167, 17214),(92, 17184) - (161, 24465),(107, 24374) - (162, 26040),(120, 25963) - (154, 4019),(138, 3990) - (259, 1850),(175, 1820) - (207, 40886),(179, 40879) - (288, 49588),(204, 49571) - (270, 32616),(226, 32607) - (318, 31489),(235, 31404) - (337, 455),(240, 359) - (270, 29508),(264, 29440) - (369, 1457),(278, 1409) -(15 rows) - -SELECT * FROM test_cube ORDER BY c~>4 LIMIT 15; -- ascending by 2nd coordinate or upper right corner - c ---------------------------- - (30333, 50),(30273, 6) - (43301, 75),(43227, 43) - (19650, 142),(19630, 51) - (2424, 160),(2424, 81) - (3449, 171),(3354, 108) - (18037, 155),(17941, 109) - (28511, 208),(28479, 114) - (19946, 217),(19941, 118) - (16906, 191),(16816, 139) - (759, 187),(662, 163) - (22684, 266),(22656, 181) - (24423, 255),(24360, 213) - (45989, 249),(45910, 222) - (11399, 377),(11360, 294) - (12162, 389),(12103, 309) -(15 rows) - -SELECT * FROM test_cube ORDER BY c~>1 DESC LIMIT 15; -- descending by 1st coordinate of lower left corner - c -------------------------------- - (50027, 49230),(49951, 49214) - (49980, 35004),(49937, 34963) - (49985, 6436),(49927, 6338) - (49999, 27218),(49908, 27176) - (49954, 1340),(49905, 1294) - (49944, 25163),(49902, 25153) - (49981, 34876),(49898, 34786) - (49957, 43390),(49897, 43384) - (49853, 18504),(49848, 18503) - (49902, 41752),(49818, 41746) - (49907, 30225),(49810, 30158) - (49843, 5175),(49808, 5145) - (49887, 24274),(49805, 24184) - (49847, 7128),(49798, 7067) - (49820, 7990),(49771, 7967) -(15 rows) - -SELECT * FROM test_cube ORDER BY c~>4 DESC LIMIT 15; -- descending by 2nd coordinate or upper right corner - c -------------------------------- - (36311, 50073),(36258, 49987) - (30746, 50040),(30727, 49992) - (2168, 50012),(2108, 49914) - (21551, 49983),(21492, 49885) - (17954, 49975),(17865, 49915) - (3531, 49962),(3463, 49934) - (19128, 49932),(19112, 49849) - (31287, 49923),(31236, 49913) - (43925, 49912),(43888, 49878) - (29261, 49910),(29247, 49818) - (14913, 49873),(14849, 49836) - (20007, 49858),(19921, 49778) - (38266, 49852),(38233, 49844) - (37595, 49849),(37581, 49834) - (46151, 49848),(46058, 49830) -(15 rows) - --- same thing for index with points -CREATE TABLE test_point(c cube); -INSERT INTO test_point(SELECT cube(array[c->1,c->2,c->3,c->4]) FROM test_cube); -CREATE INDEX ON test_point USING gist(c); -SELECT * FROM test_point ORDER BY c~>1, c~>2 LIMIT 15; -- ascending by 1st then by 2nd coordinate - c --------------------------- - (54, 38679, 3, 38602) - (83, 10271, 15, 10265) - (122, 46832, 64, 46762) - (154, 4019, 138, 3990) - (161, 24465, 107, 24374) - (162, 26040, 120, 25963) - (167, 17214, 92, 17184) - (207, 40886, 179, 40879) - (259, 1850, 175, 1820) - (270, 29508, 264, 29440) - (270, 32616, 226, 32607) - (288, 49588, 204, 49571) - (318, 31489, 235, 31404) - (326, 18837, 285, 18817) - (337, 455, 240, 359) -(15 rows) - -SELECT * FROM test_point ORDER BY c~>4 DESC LIMIT 15; -- descending by 1st coordinate - c ------------------------------- - (30746, 50040, 30727, 49992) - (36311, 50073, 36258, 49987) - (3531, 49962, 3463, 49934) - (17954, 49975, 17865, 49915) - (2168, 50012, 2108, 49914) - (31287, 49923, 31236, 49913) - (21551, 49983, 21492, 49885) - (43925, 49912, 43888, 49878) - (19128, 49932, 19112, 49849) - (38266, 49852, 38233, 49844) - (14913, 49873, 14849, 49836) - (37595, 49849, 37581, 49834) - (46151, 49848, 46058, 49830) - (29261, 49910, 29247, 49818) - (19233, 49824, 19185, 49794) -(15 rows) - diff --git a/contrib/cube/expected/cube_2.out b/contrib/cube/expected/cube_2.out index fef749c698..eaeae45fca 100644 --- a/contrib/cube/expected/cube_2.out +++ b/contrib/cube/expected/cube_2.out @@ -126,16 +126,34 @@ SELECT '-1.0e-7'::cube AS cube; (-1e-007) (1 row) -SELECT '1e-700'::cube AS cube; - cube ------- - (0) +SELECT '1e-300'::cube AS cube; + cube +---------- + (1e-300) (1 row) -SELECT '-1e-700'::cube AS cube; - cube ------- - (0) +SELECT '-1e-300'::cube AS cube; + cube +----------- + (-1e-300) +(1 row) + +SELECT 'infinity'::cube AS cube; + cube +------------ + (Infinity) +(1 row) + +SELECT '-infinity'::cube AS cube; + cube +------------- + (-Infinity) +(1 row) + +SELECT 'NaN'::cube AS cube; + cube +------- + (NaN) (1 row) SELECT '1234567890123456'::cube AS cube; @@ -175,6 +193,12 @@ SELECT '-.1234567890123456'::cube AS cube; (1 row) -- simple lists (points) +SELECT '()'::cube AS cube; + cube +------ + () +(1 row) + SELECT '1,2'::cube AS cube; cube -------- @@ -200,6 +224,12 @@ SELECT '(1,2,3,4,5)'::cube AS cube; (1 row) -- double lists (cubes) +SELECT '(),()'::cube AS cube; + cube +------ + () +(1 row) + SELECT '(0),(0)'::cube AS cube; cube ------ @@ -250,146 +280,145 @@ SELECT '[(0,0,0,0),(1,0,0,0)]'::cube AS cube; -- invalid input: parse errors SELECT ''::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT ''::cube AS cube; ^ DETAIL: syntax error at end of input SELECT 'ABC'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT 'ABC'::cube AS cube; ^ DETAIL: syntax error at or near "A" -SELECT '()'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '()'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" SELECT '[]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[]'::cube AS cube; ^ DETAIL: syntax error at or near "]" SELECT '[()]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[()]'::cube AS cube; ^ -DETAIL: syntax error at or near ")" +DETAIL: syntax error at or near "]" SELECT '[(1)]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1)]'::cube AS cube; ^ DETAIL: syntax error at or near "]" SELECT '[(1),]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1),]'::cube AS cube; ^ DETAIL: syntax error at or near "]" SELECT '[(1),2]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1),2]'::cube AS cube; ^ DETAIL: syntax error at or near "2" SELECT '[(1),(2),(3)]'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1),(2),(3)]'::cube AS cube; ^ DETAIL: syntax error at or near "," SELECT '1,'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,'::cube AS cube; ^ DETAIL: syntax error at end of input SELECT '1,2,'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,2,'::cube AS cube; ^ DETAIL: syntax error at end of input SELECT '1,,2'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,,2'::cube AS cube; ^ DETAIL: syntax error at or near "," SELECT '(1,)'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,)'::cube AS cube; ^ DETAIL: syntax error at or near ")" SELECT '(1,2,)'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2,)'::cube AS cube; ^ DETAIL: syntax error at or near ")" SELECT '(1,,2)'::cube AS cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,,2)'::cube AS cube; ^ DETAIL: syntax error at or near "," -- invalid input: semantic errors and trailing garbage SELECT '[(1),(2)],'::cube AS cube; -- 0 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1),(2)],'::cube AS cube; ^ DETAIL: syntax error at or near "," SELECT '[(1,2,3),(2,3)]'::cube AS cube; -- 1 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1,2,3),(2,3)]'::cube AS cube; ^ DETAIL: Different point dimensions in (1,2,3) and (2,3). SELECT '[(1,2),(1,2,3)]'::cube AS cube; -- 1 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '[(1,2),(1,2,3)]'::cube AS cube; ^ DETAIL: Different point dimensions in (1,2) and (1,2,3). SELECT '(1),(2),'::cube AS cube; -- 2 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1),(2),'::cube AS cube; ^ DETAIL: syntax error at or near "," SELECT '(1,2,3),(2,3)'::cube AS cube; -- 3 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2,3),(2,3)'::cube AS cube; ^ DETAIL: Different point dimensions in (1,2,3) and (2,3). SELECT '(1,2),(1,2,3)'::cube AS cube; -- 3 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2),(1,2,3)'::cube AS cube; ^ DETAIL: Different point dimensions in (1,2) and (1,2,3). SELECT '(1,2,3)ab'::cube AS cube; -- 4 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2,3)ab'::cube AS cube; ^ DETAIL: syntax error at or near "a" SELECT '(1,2,3)a'::cube AS cube; -- 5 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2,3)a'::cube AS cube; ^ DETAIL: syntax error at or near "a" SELECT '(1,2)('::cube AS cube; -- 5 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '(1,2)('::cube AS cube; ^ DETAIL: syntax error at or near "(" SELECT '1,2ab'::cube AS cube; -- 6 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,2ab'::cube AS cube; ^ DETAIL: syntax error at or near "a" SELECT '1 e7'::cube AS cube; -- 6 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1 e7'::cube AS cube; ^ DETAIL: syntax error at or near "e" SELECT '1,2a'::cube AS cube; -- 7 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1,2a'::cube AS cube; ^ DETAIL: syntax error at or near "a" SELECT '1..2'::cube AS cube; -- 7 -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: SELECT '1..2'::cube AS cube; ^ DETAIL: syntax error at or near ".2" +SELECT '-1e-700'::cube AS cube; -- out of range +ERROR: "-1e-700" is out of range for type double precision +LINE 1: SELECT '-1e-700'::cube AS cube; + ^ -- -- Testing building cubes from float8 values -- @@ -556,12 +585,12 @@ SELECT cube(cube(1,2), 42, 24); -- cube_c_f8_f8 -- Testing limit of CUBE_MAX_DIM dimensions check in cube_in. -- select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)'::cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0... ^ DETAIL: A cube cannot have more than 100 dimensions. select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0),(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)'::cube; -ERROR: bad cube representation +ERROR: invalid input syntax for cube LINE 1: select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0... ^ DETAIL: A cube cannot have more than 100 dimensions. diff --git a/contrib/cube/expected/cube_3.out b/contrib/cube/expected/cube_3.out deleted file mode 100644 index 31d2d1a64e..0000000000 --- a/contrib/cube/expected/cube_3.out +++ /dev/null @@ -1,1710 +0,0 @@ --- --- Test cube datatype --- -CREATE EXTENSION cube; --- --- testing the input and output functions --- --- Any number (a one-dimensional point) -SELECT '1'::cube AS cube; - cube ------- - (1) -(1 row) - -SELECT '-1'::cube AS cube; - cube ------- - (-1) -(1 row) - -SELECT '1.'::cube AS cube; - cube ------- - (1) -(1 row) - -SELECT '-1.'::cube AS cube; - cube ------- - (-1) -(1 row) - -SELECT '.1'::cube AS cube; - cube -------- - (0.1) -(1 row) - -SELECT '-.1'::cube AS cube; - cube --------- - (-0.1) -(1 row) - -SELECT '1.0'::cube AS cube; - cube ------- - (1) -(1 row) - -SELECT '-1.0'::cube AS cube; - cube ------- - (-1) -(1 row) - -SELECT '1e27'::cube AS cube; - cube ----------- - (1e+027) -(1 row) - -SELECT '-1e27'::cube AS cube; - cube ------------ - (-1e+027) -(1 row) - -SELECT '1.0e27'::cube AS cube; - cube ----------- - (1e+027) -(1 row) - -SELECT '-1.0e27'::cube AS cube; - cube ------------ - (-1e+027) -(1 row) - -SELECT '1e+27'::cube AS cube; - cube ----------- - (1e+027) -(1 row) - -SELECT '-1e+27'::cube AS cube; - cube ------------ - (-1e+027) -(1 row) - -SELECT '1.0e+27'::cube AS cube; - cube ----------- - (1e+027) -(1 row) - -SELECT '-1.0e+27'::cube AS cube; - cube ------------ - (-1e+027) -(1 row) - -SELECT '1e-7'::cube AS cube; - cube ----------- - (1e-007) -(1 row) - -SELECT '-1e-7'::cube AS cube; - cube ------------ - (-1e-007) -(1 row) - -SELECT '1.0e-7'::cube AS cube; - cube ----------- - (1e-007) -(1 row) - -SELECT '-1.0e-7'::cube AS cube; - cube ------------ - (-1e-007) -(1 row) - -SELECT '1e-700'::cube AS cube; - cube ------- - (0) -(1 row) - -SELECT '-1e-700'::cube AS cube; - cube ------- - (-0) -(1 row) - -SELECT '1234567890123456'::cube AS cube; - cube -------------------------- - (1.23456789012346e+015) -(1 row) - -SELECT '+1234567890123456'::cube AS cube; - cube -------------------------- - (1.23456789012346e+015) -(1 row) - -SELECT '-1234567890123456'::cube AS cube; - cube --------------------------- - (-1.23456789012346e+015) -(1 row) - -SELECT '.1234567890123456'::cube AS cube; - cube ---------------------- - (0.123456789012346) -(1 row) - -SELECT '+.1234567890123456'::cube AS cube; - cube ---------------------- - (0.123456789012346) -(1 row) - -SELECT '-.1234567890123456'::cube AS cube; - cube ----------------------- - (-0.123456789012346) -(1 row) - --- simple lists (points) -SELECT '1,2'::cube AS cube; - cube --------- - (1, 2) -(1 row) - -SELECT '(1,2)'::cube AS cube; - cube --------- - (1, 2) -(1 row) - -SELECT '1,2,3,4,5'::cube AS cube; - cube ------------------ - (1, 2, 3, 4, 5) -(1 row) - -SELECT '(1,2,3,4,5)'::cube AS cube; - cube ------------------ - (1, 2, 3, 4, 5) -(1 row) - --- double lists (cubes) -SELECT '(0),(0)'::cube AS cube; - cube ------- - (0) -(1 row) - -SELECT '(0),(1)'::cube AS cube; - cube ---------- - (0),(1) -(1 row) - -SELECT '[(0),(0)]'::cube AS cube; - cube ------- - (0) -(1 row) - -SELECT '[(0),(1)]'::cube AS cube; - cube ---------- - (0),(1) -(1 row) - -SELECT '(0,0,0,0),(0,0,0,0)'::cube AS cube; - cube --------------- - (0, 0, 0, 0) -(1 row) - -SELECT '(0,0,0,0),(1,0,0,0)'::cube AS cube; - cube ---------------------------- - (0, 0, 0, 0),(1, 0, 0, 0) -(1 row) - -SELECT '[(0,0,0,0),(0,0,0,0)]'::cube AS cube; - cube --------------- - (0, 0, 0, 0) -(1 row) - -SELECT '[(0,0,0,0),(1,0,0,0)]'::cube AS cube; - cube ---------------------------- - (0, 0, 0, 0),(1, 0, 0, 0) -(1 row) - --- invalid input: parse errors -SELECT ''::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT ''::cube AS cube; - ^ -DETAIL: syntax error at end of input -SELECT 'ABC'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT 'ABC'::cube AS cube; - ^ -DETAIL: syntax error at or near "A" -SELECT '()'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '()'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" -SELECT '[]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[]'::cube AS cube; - ^ -DETAIL: syntax error at or near "]" -SELECT '[()]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[()]'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" -SELECT '[(1)]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[(1)]'::cube AS cube; - ^ -DETAIL: syntax error at or near "]" -SELECT '[(1),]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[(1),]'::cube AS cube; - ^ -DETAIL: syntax error at or near "]" -SELECT '[(1),2]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[(1),2]'::cube AS cube; - ^ -DETAIL: syntax error at or near "2" -SELECT '[(1),(2),(3)]'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '[(1),(2),(3)]'::cube AS cube; - ^ -DETAIL: syntax error at or near "," -SELECT '1,'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '1,'::cube AS cube; - ^ -DETAIL: syntax error at end of input -SELECT '1,2,'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '1,2,'::cube AS cube; - ^ -DETAIL: syntax error at end of input -SELECT '1,,2'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '1,,2'::cube AS cube; - ^ -DETAIL: syntax error at or near "," -SELECT '(1,)'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '(1,)'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" -SELECT '(1,2,)'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '(1,2,)'::cube AS cube; - ^ -DETAIL: syntax error at or near ")" -SELECT '(1,,2)'::cube AS cube; -ERROR: bad cube representation -LINE 1: SELECT '(1,,2)'::cube AS cube; - ^ -DETAIL: syntax error at or near "," --- invalid input: semantic errors and trailing garbage -SELECT '[(1),(2)],'::cube AS cube; -- 0 -ERROR: bad cube representation -LINE 1: SELECT '[(1),(2)],'::cube AS cube; - ^ -DETAIL: syntax error at or near "," -SELECT '[(1,2,3),(2,3)]'::cube AS cube; -- 1 -ERROR: bad cube representation -LINE 1: SELECT '[(1,2,3),(2,3)]'::cube AS cube; - ^ -DETAIL: Different point dimensions in (1,2,3) and (2,3). -SELECT '[(1,2),(1,2,3)]'::cube AS cube; -- 1 -ERROR: bad cube representation -LINE 1: SELECT '[(1,2),(1,2,3)]'::cube AS cube; - ^ -DETAIL: Different point dimensions in (1,2) and (1,2,3). -SELECT '(1),(2),'::cube AS cube; -- 2 -ERROR: bad cube representation -LINE 1: SELECT '(1),(2),'::cube AS cube; - ^ -DETAIL: syntax error at or near "," -SELECT '(1,2,3),(2,3)'::cube AS cube; -- 3 -ERROR: bad cube representation -LINE 1: SELECT '(1,2,3),(2,3)'::cube AS cube; - ^ -DETAIL: Different point dimensions in (1,2,3) and (2,3). -SELECT '(1,2),(1,2,3)'::cube AS cube; -- 3 -ERROR: bad cube representation -LINE 1: SELECT '(1,2),(1,2,3)'::cube AS cube; - ^ -DETAIL: Different point dimensions in (1,2) and (1,2,3). -SELECT '(1,2,3)ab'::cube AS cube; -- 4 -ERROR: bad cube representation -LINE 1: SELECT '(1,2,3)ab'::cube AS cube; - ^ -DETAIL: syntax error at or near "a" -SELECT '(1,2,3)a'::cube AS cube; -- 5 -ERROR: bad cube representation -LINE 1: SELECT '(1,2,3)a'::cube AS cube; - ^ -DETAIL: syntax error at or near "a" -SELECT '(1,2)('::cube AS cube; -- 5 -ERROR: bad cube representation -LINE 1: SELECT '(1,2)('::cube AS cube; - ^ -DETAIL: syntax error at or near "(" -SELECT '1,2ab'::cube AS cube; -- 6 -ERROR: bad cube representation -LINE 1: SELECT '1,2ab'::cube AS cube; - ^ -DETAIL: syntax error at or near "a" -SELECT '1 e7'::cube AS cube; -- 6 -ERROR: bad cube representation -LINE 1: SELECT '1 e7'::cube AS cube; - ^ -DETAIL: syntax error at or near "e" -SELECT '1,2a'::cube AS cube; -- 7 -ERROR: bad cube representation -LINE 1: SELECT '1,2a'::cube AS cube; - ^ -DETAIL: syntax error at or near "a" -SELECT '1..2'::cube AS cube; -- 7 -ERROR: bad cube representation -LINE 1: SELECT '1..2'::cube AS cube; - ^ -DETAIL: syntax error at or near ".2" --- --- Testing building cubes from float8 values --- -SELECT cube(0::float8); - cube ------- - (0) -(1 row) - -SELECT cube(1::float8); - cube ------- - (1) -(1 row) - -SELECT cube(1,2); - cube ---------- - (1),(2) -(1 row) - -SELECT cube(cube(1,2),3); - cube ---------------- - (1, 3),(2, 3) -(1 row) - -SELECT cube(cube(1,2),3,4); - cube ---------------- - (1, 3),(2, 4) -(1 row) - -SELECT cube(cube(cube(1,2),3,4),5); - cube ---------------------- - (1, 3, 5),(2, 4, 5) -(1 row) - -SELECT cube(cube(cube(1,2),3,4),5,6); - cube ---------------------- - (1, 3, 5),(2, 4, 6) -(1 row) - --- --- Test that the text -> cube cast was installed. --- -SELECT '(0)'::text::cube; - cube ------- - (0) -(1 row) - --- --- Test the float[] -> cube cast --- -SELECT cube('{0,1,2}'::float[], '{3,4,5}'::float[]); - cube ---------------------- - (0, 1, 2),(3, 4, 5) -(1 row) - -SELECT cube('{0,1,2}'::float[], '{3}'::float[]); -ERROR: UR and LL arrays must be of same length -SELECT cube(NULL::float[], '{3}'::float[]); - cube ------- - -(1 row) - -SELECT cube('{0,1,2}'::float[]); - cube ------------ - (0, 1, 2) -(1 row) - -SELECT cube_subset(cube('(1,3,5),(6,7,8)'), ARRAY[3,2,1,1]); - cube_subset ---------------------------- - (5, 3, 1, 1),(8, 7, 6, 6) -(1 row) - -SELECT cube_subset(cube('(1,3,5),(1,3,5)'), ARRAY[3,2,1,1]); - cube_subset --------------- - (5, 3, 1, 1) -(1 row) - -SELECT cube_subset(cube('(1,3,5),(6,7,8)'), ARRAY[4,0]); -ERROR: Index out of bounds -SELECT cube_subset(cube('(6,7,8),(6,7,8)'), ARRAY[4,0]); -ERROR: Index out of bounds --- --- Test point processing --- -SELECT cube('(1,2),(1,2)'); -- cube_in - cube --------- - (1, 2) -(1 row) - -SELECT cube('{0,1,2}'::float[], '{0,1,2}'::float[]); -- cube_a_f8_f8 - cube ------------ - (0, 1, 2) -(1 row) - -SELECT cube('{5,6,7,8}'::float[]); -- cube_a_f8 - cube --------------- - (5, 6, 7, 8) -(1 row) - -SELECT cube(1.37); -- cube_f8 - cube --------- - (1.37) -(1 row) - -SELECT cube(1.37, 1.37); -- cube_f8_f8 - cube --------- - (1.37) -(1 row) - -SELECT cube(cube(1,1), 42); -- cube_c_f8 - cube ---------- - (1, 42) -(1 row) - -SELECT cube(cube(1,2), 42); -- cube_c_f8 - cube ------------------ - (1, 42),(2, 42) -(1 row) - -SELECT cube(cube(1,1), 42, 42); -- cube_c_f8_f8 - cube ---------- - (1, 42) -(1 row) - -SELECT cube(cube(1,1), 42, 24); -- cube_c_f8_f8 - cube ------------------ - (1, 42),(1, 24) -(1 row) - -SELECT cube(cube(1,2), 42, 42); -- cube_c_f8_f8 - cube ------------------ - (1, 42),(2, 42) -(1 row) - -SELECT cube(cube(1,2), 42, 24); -- cube_c_f8_f8 - cube ------------------ - (1, 42),(2, 24) -(1 row) - --- --- Testing limit of CUBE_MAX_DIM dimensions check in cube_in. --- -select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)'::cube; -ERROR: bad cube representation -LINE 1: select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0... - ^ -DETAIL: A cube cannot have more than 100 dimensions. -select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0),(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)'::cube; -ERROR: bad cube representation -LINE 1: select '(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0... - ^ -DETAIL: A cube cannot have more than 100 dimensions. --- --- testing the operators --- --- equality/inequality: --- -SELECT '24, 33.20'::cube = '24, 33.20'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '24, 33.20'::cube != '24, 33.20'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '24, 33.20'::cube = '24, 33.21'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '24, 33.20'::cube != '24, 33.21'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0),(3,1)'::cube = '(2,0,0,0,0),(3,1,0,0,0)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0),(3,1)'::cube = '(2,0,0,0,0),(3,1,0,0,1)'::cube AS bool; - bool ------- - f -(1 row) - --- "lower than" / "greater than" --- (these operators are not useful for anything but ordering) --- -SELECT '1'::cube > '2'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '1'::cube < '2'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '1,1'::cube > '1,2'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '1,1'::cube < '1,2'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0),(3,1)'::cube > '(2,0,0,0,0),(3,1,0,0,1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0),(3,1)'::cube < '(2,0,0,0,0),(3,1,0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0),(3,1)'::cube > '(2,0,0,0,1),(3,1,0,0,0)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0),(3,1)'::cube < '(2,0,0,0,1),(3,1,0,0,0)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0),(3,1)'::cube > '(2,0,0,0,0),(3,1,0,0,0)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0),(3,1)'::cube < '(2,0,0,0,0),(3,1,0,0,0)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0,0,0,0),(3,1,0,0,1)'::cube > '(2,0),(3,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0,0,0,0),(3,1,0,0,1)'::cube < '(2,0),(3,1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0,0,0,1),(3,1,0,0,0)'::cube > '(2,0),(3,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0,0,0,1),(3,1,0,0,0)'::cube < '(2,0),(3,1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(2,0,0,0,0),(3,1,0,0,0)'::cube > '(2,0),(3,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(2,0,0,0,0),(3,1,0,0,0)'::cube < '(2,0),(3,1)'::cube AS bool; - bool ------- - f -(1 row) - --- "overlap" --- -SELECT '1'::cube && '1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '1'::cube && '2'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '1,1,1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '[(1,1,1),(2,2,2)]'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '[(1,1),(2,2)]'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '[(-1,-1,-1),(1,1,1)]'::cube && '[(2,1,1),(2,2,2)]'::cube AS bool; - bool ------- - f -(1 row) - --- "contained in" (the left operand is the cube entirely enclosed by --- the right operand): --- -SELECT '0'::cube <@ '0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,0'::cube <@ '0,0,0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0'::cube <@ '0,0,1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,0'::cube <@ '0,0,1'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '1,0,0'::cube <@ '0,0,1'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(1,0,0),(0,0,1)'::cube <@ '(1,0,0),(0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(1,0,0),(0,0,1)'::cube <@ '(-1,-1,-1),(1,1,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(1,0,0),(0,0,1)'::cube <@ '(-1,-1,-1,-1),(1,1,1,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '1'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '-1'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube <@ '(-1,-1),(1,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-2),(1)'::cube <@ '(-1),(1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(-2),(1)'::cube <@ '(-1,-1),(1,1)'::cube AS bool; - bool ------- - f -(1 row) - --- "contains" (the left operand is the cube that entirely encloses the --- right operand) --- -SELECT '0'::cube @> '0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,0'::cube @> '0,0,0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,1'::cube @> '0,0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '0,0,1'::cube @> '0,0,0'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '0,0,1'::cube @> '1,0,0'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(1,0,0),(0,0,1)'::cube @> '(1,0,0),(0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1,-1,-1),(1,1,1)'::cube @> '(1,0,0),(0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1,-1,-1,-1),(1,1,1,1)'::cube @> '(1,0,0),(0,0,1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '0'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '-1'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1,-1),(1,1)'::cube @> '(-1),(1)'::cube AS bool; - bool ------- - t -(1 row) - -SELECT '(-1),(1)'::cube @> '(-2),(1)'::cube AS bool; - bool ------- - f -(1 row) - -SELECT '(-1,-1),(1,1)'::cube @> '(-2),(1)'::cube AS bool; - bool ------- - f -(1 row) - --- Test of distance function --- -SELECT cube_distance('(0)'::cube,'(2,2,2,2)'::cube); - cube_distance ---------------- - 4 -(1 row) - -SELECT cube_distance('(0)'::cube,'(.3,.4)'::cube); - cube_distance ---------------- - 0.5 -(1 row) - -SELECT cube_distance('(2,3,4)'::cube,'(2,3,4)'::cube); - cube_distance ---------------- - 0 -(1 row) - -SELECT cube_distance('(42,42,42,42)'::cube,'(137,137,137,137)'::cube); - cube_distance ---------------- - 190 -(1 row) - -SELECT cube_distance('(42,42,42)'::cube,'(137,137)'::cube); - cube_distance ------------------- - 140.762210837994 -(1 row) - --- Test of cube function (text to cube) --- -SELECT cube('(1,1.2)'::text); - cube ----------- - (1, 1.2) -(1 row) - -SELECT cube(NULL); - cube ------- - -(1 row) - --- Test of cube_dim function (dimensions stored in cube) --- -SELECT cube_dim('(0)'::cube); - cube_dim ----------- - 1 -(1 row) - -SELECT cube_dim('(0,0)'::cube); - cube_dim ----------- - 2 -(1 row) - -SELECT cube_dim('(0,0,0)'::cube); - cube_dim ----------- - 3 -(1 row) - -SELECT cube_dim('(42,42,42),(42,42,42)'::cube); - cube_dim ----------- - 3 -(1 row) - -SELECT cube_dim('(4,8,15,16,23),(4,8,15,16,23)'::cube); - cube_dim ----------- - 5 -(1 row) - --- Test of cube_ll_coord function (retrieves LL coodinate values) --- -SELECT cube_ll_coord('(-1,1),(2,-2)'::cube, 1); - cube_ll_coord ---------------- - -1 -(1 row) - -SELECT cube_ll_coord('(-1,1),(2,-2)'::cube, 2); - cube_ll_coord ---------------- - -2 -(1 row) - -SELECT cube_ll_coord('(-1,1),(2,-2)'::cube, 3); - cube_ll_coord ---------------- - 0 -(1 row) - -SELECT cube_ll_coord('(1,2),(1,2)'::cube, 1); - cube_ll_coord ---------------- - 1 -(1 row) - -SELECT cube_ll_coord('(1,2),(1,2)'::cube, 2); - cube_ll_coord ---------------- - 2 -(1 row) - -SELECT cube_ll_coord('(1,2),(1,2)'::cube, 3); - cube_ll_coord ---------------- - 0 -(1 row) - -SELECT cube_ll_coord('(42,137)'::cube, 1); - cube_ll_coord ---------------- - 42 -(1 row) - -SELECT cube_ll_coord('(42,137)'::cube, 2); - cube_ll_coord ---------------- - 137 -(1 row) - -SELECT cube_ll_coord('(42,137)'::cube, 3); - cube_ll_coord ---------------- - 0 -(1 row) - --- Test of cube_ur_coord function (retrieves UR coodinate values) --- -SELECT cube_ur_coord('(-1,1),(2,-2)'::cube, 1); - cube_ur_coord ---------------- - 2 -(1 row) - -SELECT cube_ur_coord('(-1,1),(2,-2)'::cube, 2); - cube_ur_coord ---------------- - 1 -(1 row) - -SELECT cube_ur_coord('(-1,1),(2,-2)'::cube, 3); - cube_ur_coord ---------------- - 0 -(1 row) - -SELECT cube_ur_coord('(1,2),(1,2)'::cube, 1); - cube_ur_coord ---------------- - 1 -(1 row) - -SELECT cube_ur_coord('(1,2),(1,2)'::cube, 2); - cube_ur_coord ---------------- - 2 -(1 row) - -SELECT cube_ur_coord('(1,2),(1,2)'::cube, 3); - cube_ur_coord ---------------- - 0 -(1 row) - -SELECT cube_ur_coord('(42,137)'::cube, 1); - cube_ur_coord ---------------- - 42 -(1 row) - -SELECT cube_ur_coord('(42,137)'::cube, 2); - cube_ur_coord ---------------- - 137 -(1 row) - -SELECT cube_ur_coord('(42,137)'::cube, 3); - cube_ur_coord ---------------- - 0 -(1 row) - --- Test of cube_is_point --- -SELECT cube_is_point('(0)'::cube); - cube_is_point ---------------- - t -(1 row) - -SELECT cube_is_point('(0,1,2)'::cube); - cube_is_point ---------------- - t -(1 row) - -SELECT cube_is_point('(0,1,2),(0,1,2)'::cube); - cube_is_point ---------------- - t -(1 row) - -SELECT cube_is_point('(0,1,2),(-1,1,2)'::cube); - cube_is_point ---------------- - f -(1 row) - -SELECT cube_is_point('(0,1,2),(0,-1,2)'::cube); - cube_is_point ---------------- - f -(1 row) - -SELECT cube_is_point('(0,1,2),(0,1,-2)'::cube); - cube_is_point ---------------- - f -(1 row) - --- Test of cube_enlarge (enlarging and shrinking cubes) --- -SELECT cube_enlarge('(0)'::cube, 0, 0); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 0, 1); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 0, 2); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(2),(-2)'::cube, 0, 4); - cube_enlarge --------------- - (-2),(2) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 1, 0); - cube_enlarge --------------- - (-1),(1) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 1, 1); - cube_enlarge --------------- - (-1),(1) -(1 row) - -SELECT cube_enlarge('(0)'::cube, 1, 2); - cube_enlarge ------------------ - (-1, -1),(1, 1) -(1 row) - -SELECT cube_enlarge('(2),(-2)'::cube, 1, 4); - cube_enlarge -------------------------------- - (-3, -1, -1, -1),(3, 1, 1, 1) -(1 row) - -SELECT cube_enlarge('(0)'::cube, -1, 0); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(0)'::cube, -1, 1); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(0)'::cube, -1, 2); - cube_enlarge --------------- - (0) -(1 row) - -SELECT cube_enlarge('(2),(-2)'::cube, -1, 4); - cube_enlarge --------------- - (-1),(1) -(1 row) - -SELECT cube_enlarge('(0,0,0)'::cube, 1, 0); - cube_enlarge ------------------------- - (-1, -1, -1),(1, 1, 1) -(1 row) - -SELECT cube_enlarge('(0,0,0)'::cube, 1, 2); - cube_enlarge ------------------------- - (-1, -1, -1),(1, 1, 1) -(1 row) - -SELECT cube_enlarge('(2,-2),(-3,7)'::cube, 1, 2); - cube_enlarge ------------------ - (-4, -3),(3, 8) -(1 row) - -SELECT cube_enlarge('(2,-2),(-3,7)'::cube, 3, 2); - cube_enlarge ------------------- - (-6, -5),(5, 10) -(1 row) - -SELECT cube_enlarge('(2,-2),(-3,7)'::cube, -1, 2); - cube_enlarge ------------------ - (-2, -1),(1, 6) -(1 row) - -SELECT cube_enlarge('(2,-2),(-3,7)'::cube, -3, 2); - cube_enlarge ---------------------- - (-0.5, 1),(-0.5, 4) -(1 row) - -SELECT cube_enlarge('(42,-23,-23),(42,23,23)'::cube, -23, 5); - cube_enlarge --------------- - (42, 0, 0) -(1 row) - -SELECT cube_enlarge('(42,-23,-23),(42,23,23)'::cube, -24, 5); - cube_enlarge --------------- - (42, 0, 0) -(1 row) - --- Test of cube_union (MBR for two cubes) --- -SELECT cube_union('(1,2),(3,4)'::cube, '(5,6,7),(8,9,10)'::cube); - cube_union ----------------------- - (1, 2, 0),(8, 9, 10) -(1 row) - -SELECT cube_union('(1,2)'::cube, '(4,2,0,0)'::cube); - cube_union ---------------------------- - (1, 2, 0, 0),(4, 2, 0, 0) -(1 row) - -SELECT cube_union('(1,2),(1,2)'::cube, '(4,2),(4,2)'::cube); - cube_union ---------------- - (1, 2),(4, 2) -(1 row) - -SELECT cube_union('(1,2),(1,2)'::cube, '(1,2),(1,2)'::cube); - cube_union ------------- - (1, 2) -(1 row) - -SELECT cube_union('(1,2),(1,2)'::cube, '(1,2,0),(1,2,0)'::cube); - cube_union ------------- - (1, 2, 0) -(1 row) - --- Test of cube_inter --- -SELECT cube_inter('(1,2),(10,11)'::cube, '(3,4), (16,15)'::cube); -- intersects - cube_inter ------------------ - (3, 4),(10, 11) -(1 row) - -SELECT cube_inter('(1,2),(10,11)'::cube, '(3,4), (6,5)'::cube); -- includes - cube_inter ---------------- - (3, 4),(6, 5) -(1 row) - -SELECT cube_inter('(1,2),(10,11)'::cube, '(13,14), (16,15)'::cube); -- no intersection - cube_inter -------------------- - (13, 14),(10, 11) -(1 row) - -SELECT cube_inter('(1,2),(10,11)'::cube, '(3,14), (16,15)'::cube); -- no intersection, but one dimension intersects - cube_inter ------------------- - (3, 14),(10, 11) -(1 row) - -SELECT cube_inter('(1,2),(10,11)'::cube, '(10,11), (16,15)'::cube); -- point intersection - cube_inter ------------- - (10, 11) -(1 row) - -SELECT cube_inter('(1,2,3)'::cube, '(1,2,3)'::cube); -- point args - cube_inter ------------- - (1, 2, 3) -(1 row) - -SELECT cube_inter('(1,2,3)'::cube, '(5,6,3)'::cube); -- point args - cube_inter ---------------------- - (5, 6, 3),(1, 2, 3) -(1 row) - --- Test of cube_size --- -SELECT cube_size('(4,8),(15,16)'::cube); - cube_size ------------ - 88 -(1 row) - -SELECT cube_size('(42,137)'::cube); - cube_size ------------ - 0 -(1 row) - --- Test of distances --- -SELECT cube_distance('(1,1)'::cube, '(4,5)'::cube); - cube_distance ---------------- - 5 -(1 row) - -SELECT '(1,1)'::cube <-> '(4,5)'::cube as d_e; - d_e ------ - 5 -(1 row) - -SELECT distance_chebyshev('(1,1)'::cube, '(4,5)'::cube); - distance_chebyshev --------------------- - 4 -(1 row) - -SELECT '(1,1)'::cube <=> '(4,5)'::cube as d_c; - d_c ------ - 4 -(1 row) - -SELECT distance_taxicab('(1,1)'::cube, '(4,5)'::cube); - distance_taxicab ------------------- - 7 -(1 row) - -SELECT '(1,1)'::cube <#> '(4,5)'::cube as d_t; - d_t ------ - 7 -(1 row) - --- zero for overlapping -SELECT cube_distance('(2,2),(10,10)'::cube, '(0,0),(5,5)'::cube); - cube_distance ---------------- - 0 -(1 row) - -SELECT distance_chebyshev('(2,2),(10,10)'::cube, '(0,0),(5,5)'::cube); - distance_chebyshev --------------------- - 0 -(1 row) - -SELECT distance_taxicab('(2,2),(10,10)'::cube, '(0,0),(5,5)'::cube); - distance_taxicab ------------------- - 0 -(1 row) - --- coordinate access -SELECT cube(array[10,20,30], array[40,50,60])->1; - ?column? ----------- - 10 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])->1; - ?column? ----------- - 40 -(1 row) - -SELECT cube(array[10,20,30], array[40,50,60])->6; - ?column? ----------- - 60 -(1 row) - -SELECT cube(array[10,20,30], array[40,50,60])->0; -ERROR: cube index 0 is out of bounds -SELECT cube(array[10,20,30], array[40,50,60])->7; -ERROR: cube index 7 is out of bounds -SELECT cube(array[10,20,30], array[40,50,60])->-1; -ERROR: cube index -1 is out of bounds -SELECT cube(array[10,20,30], array[40,50,60])->-6; -ERROR: cube index -6 is out of bounds -SELECT cube(array[10,20,30])->3; - ?column? ----------- - 30 -(1 row) - -SELECT cube(array[10,20,30])->6; - ?column? ----------- - 30 -(1 row) - -SELECT cube(array[10,20,30])->-6; -ERROR: cube index -6 is out of bounds --- "normalized" coordinate access -SELECT cube(array[10,20,30], array[40,50,60])~>1; - ?column? ----------- - 10 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>1; - ?column? ----------- - 10 -(1 row) - -SELECT cube(array[10,20,30], array[40,50,60])~>2; - ?column? ----------- - 20 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>2; - ?column? ----------- - 20 -(1 row) - -SELECT cube(array[10,20,30], array[40,50,60])~>3; - ?column? ----------- - 30 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>3; - ?column? ----------- - 30 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>0; -ERROR: cube index 0 is out of bounds -SELECT cube(array[40,50,60], array[10,20,30])~>4; - ?column? ----------- - 40 -(1 row) - -SELECT cube(array[40,50,60], array[10,20,30])~>(-1); -ERROR: cube index -1 is out of bounds --- Load some example data and build the index --- -CREATE TABLE test_cube (c cube); -\copy test_cube from 'data/test_cube.data' -CREATE INDEX test_cube_ix ON test_cube USING gist (c); -SELECT * FROM test_cube WHERE c && '(3000,1000),(0,0)' ORDER BY c; - c --------------------------- - (337, 455),(240, 359) - (759, 187),(662, 163) - (1444, 403),(1346, 344) - (1594, 1043),(1517, 971) - (2424, 160),(2424, 81) -(5 rows) - --- Test sorting -SELECT * FROM test_cube WHERE c && '(3000,1000),(0,0)' GROUP BY c ORDER BY c; - c --------------------------- - (337, 455),(240, 359) - (759, 187),(662, 163) - (1444, 403),(1346, 344) - (1594, 1043),(1517, 971) - (2424, 160),(2424, 81) -(5 rows) - --- kNN with index -SELECT *, c <-> '(100, 100),(500, 500)'::cube as dist FROM test_cube ORDER BY c <-> '(100, 100),(500, 500)'::cube LIMIT 5; - c | dist --------------------------+------------------ - (337, 455),(240, 359) | 0 - (759, 187),(662, 163) | 162 - (948, 1201),(907, 1156) | 772.000647668122 - (1444, 403),(1346, 344) | 846 - (369, 1457),(278, 1409) | 909 -(5 rows) - -SELECT *, c <=> '(100, 100),(500, 500)'::cube as dist FROM test_cube ORDER BY c <=> '(100, 100),(500, 500)'::cube LIMIT 5; - c | dist --------------------------+------ - (337, 455),(240, 359) | 0 - (759, 187),(662, 163) | 162 - (948, 1201),(907, 1156) | 656 - (1444, 403),(1346, 344) | 846 - (369, 1457),(278, 1409) | 909 -(5 rows) - -SELECT *, c <#> '(100, 100),(500, 500)'::cube as dist FROM test_cube ORDER BY c <#> '(100, 100),(500, 500)'::cube LIMIT 5; - c | dist --------------------------+------ - (337, 455),(240, 359) | 0 - (759, 187),(662, 163) | 162 - (1444, 403),(1346, 344) | 846 - (369, 1457),(278, 1409) | 909 - (948, 1201),(907, 1156) | 1063 -(5 rows) - --- kNN-based sorting -SELECT * FROM test_cube ORDER BY c~>1 LIMIT 15; -- ascending by 1st coordinate of lower left corner - c ---------------------------- - (54, 38679),(3, 38602) - (83, 10271),(15, 10265) - (122, 46832),(64, 46762) - (167, 17214),(92, 17184) - (161, 24465),(107, 24374) - (162, 26040),(120, 25963) - (154, 4019),(138, 3990) - (259, 1850),(175, 1820) - (207, 40886),(179, 40879) - (288, 49588),(204, 49571) - (270, 32616),(226, 32607) - (318, 31489),(235, 31404) - (337, 455),(240, 359) - (270, 29508),(264, 29440) - (369, 1457),(278, 1409) -(15 rows) - -SELECT * FROM test_cube ORDER BY c~>4 LIMIT 15; -- ascending by 2nd coordinate or upper right corner - c ---------------------------- - (30333, 50),(30273, 6) - (43301, 75),(43227, 43) - (19650, 142),(19630, 51) - (2424, 160),(2424, 81) - (3449, 171),(3354, 108) - (18037, 155),(17941, 109) - (28511, 208),(28479, 114) - (19946, 217),(19941, 118) - (16906, 191),(16816, 139) - (759, 187),(662, 163) - (22684, 266),(22656, 181) - (24423, 255),(24360, 213) - (45989, 249),(45910, 222) - (11399, 377),(11360, 294) - (12162, 389),(12103, 309) -(15 rows) - -SELECT * FROM test_cube ORDER BY c~>1 DESC LIMIT 15; -- descending by 1st coordinate of lower left corner - c -------------------------------- - (50027, 49230),(49951, 49214) - (49980, 35004),(49937, 34963) - (49985, 6436),(49927, 6338) - (49999, 27218),(49908, 27176) - (49954, 1340),(49905, 1294) - (49944, 25163),(49902, 25153) - (49981, 34876),(49898, 34786) - (49957, 43390),(49897, 43384) - (49853, 18504),(49848, 18503) - (49902, 41752),(49818, 41746) - (49907, 30225),(49810, 30158) - (49843, 5175),(49808, 5145) - (49887, 24274),(49805, 24184) - (49847, 7128),(49798, 7067) - (49820, 7990),(49771, 7967) -(15 rows) - -SELECT * FROM test_cube ORDER BY c~>4 DESC LIMIT 15; -- descending by 2nd coordinate or upper right corner - c -------------------------------- - (36311, 50073),(36258, 49987) - (30746, 50040),(30727, 49992) - (2168, 50012),(2108, 49914) - (21551, 49983),(21492, 49885) - (17954, 49975),(17865, 49915) - (3531, 49962),(3463, 49934) - (19128, 49932),(19112, 49849) - (31287, 49923),(31236, 49913) - (43925, 49912),(43888, 49878) - (29261, 49910),(29247, 49818) - (14913, 49873),(14849, 49836) - (20007, 49858),(19921, 49778) - (38266, 49852),(38233, 49844) - (37595, 49849),(37581, 49834) - (46151, 49848),(46058, 49830) -(15 rows) - --- same thing for index with points -CREATE TABLE test_point(c cube); -INSERT INTO test_point(SELECT cube(array[c->1,c->2,c->3,c->4]) FROM test_cube); -CREATE INDEX ON test_point USING gist(c); -SELECT * FROM test_point ORDER BY c~>1, c~>2 LIMIT 15; -- ascending by 1st then by 2nd coordinate - c --------------------------- - (54, 38679, 3, 38602) - (83, 10271, 15, 10265) - (122, 46832, 64, 46762) - (154, 4019, 138, 3990) - (161, 24465, 107, 24374) - (162, 26040, 120, 25963) - (167, 17214, 92, 17184) - (207, 40886, 179, 40879) - (259, 1850, 175, 1820) - (270, 29508, 264, 29440) - (270, 32616, 226, 32607) - (288, 49588, 204, 49571) - (318, 31489, 235, 31404) - (326, 18837, 285, 18817) - (337, 455, 240, 359) -(15 rows) - -SELECT * FROM test_point ORDER BY c~>4 DESC LIMIT 15; -- descending by 1st coordinate - c ------------------------------- - (30746, 50040, 30727, 49992) - (36311, 50073, 36258, 49987) - (3531, 49962, 3463, 49934) - (17954, 49975, 17865, 49915) - (2168, 50012, 2108, 49914) - (31287, 49923, 31236, 49913) - (21551, 49983, 21492, 49885) - (43925, 49912, 43888, 49878) - (19128, 49932, 19112, 49849) - (38266, 49852, 38233, 49844) - (14913, 49873, 14849, 49836) - (37595, 49849, 37581, 49834) - (46151, 49848, 46058, 49830) - (29261, 49910, 29247, 49818) - (19233, 49824, 19185, 49794) -(15 rows) - diff --git a/contrib/cube/sql/cube.sql b/contrib/cube/sql/cube.sql index e225fb7da1..cc5a1ce458 100644 --- a/contrib/cube/sql/cube.sql +++ b/contrib/cube/sql/cube.sql @@ -29,8 +29,11 @@ SELECT '1e-7'::cube AS cube; SELECT '-1e-7'::cube AS cube; SELECT '1.0e-7'::cube AS cube; SELECT '-1.0e-7'::cube AS cube; -SELECT '1e-700'::cube AS cube; -SELECT '-1e-700'::cube AS cube; +SELECT '1e-300'::cube AS cube; +SELECT '-1e-300'::cube AS cube; +SELECT 'infinity'::cube AS cube; +SELECT '-infinity'::cube AS cube; +SELECT 'NaN'::cube AS cube; SELECT '1234567890123456'::cube AS cube; SELECT '+1234567890123456'::cube AS cube; SELECT '-1234567890123456'::cube AS cube; @@ -39,12 +42,14 @@ SELECT '+.1234567890123456'::cube AS cube; SELECT '-.1234567890123456'::cube AS cube; -- simple lists (points) +SELECT '()'::cube AS cube; SELECT '1,2'::cube AS cube; SELECT '(1,2)'::cube AS cube; SELECT '1,2,3,4,5'::cube AS cube; SELECT '(1,2,3,4,5)'::cube AS cube; -- double lists (cubes) +SELECT '(),()'::cube AS cube; SELECT '(0),(0)'::cube AS cube; SELECT '(0),(1)'::cube AS cube; SELECT '[(0),(0)]'::cube AS cube; @@ -57,7 +62,6 @@ SELECT '[(0,0,0,0),(1,0,0,0)]'::cube AS cube; -- invalid input: parse errors SELECT ''::cube AS cube; SELECT 'ABC'::cube AS cube; -SELECT '()'::cube AS cube; SELECT '[]'::cube AS cube; SELECT '[()]'::cube AS cube; SELECT '[(1)]'::cube AS cube; @@ -85,6 +89,7 @@ SELECT '1,2ab'::cube AS cube; -- 6 SELECT '1 e7'::cube AS cube; -- 6 SELECT '1,2a'::cube AS cube; -- 7 SELECT '1..2'::cube AS cube; -- 7 +SELECT '-1e-700'::cube AS cube; -- out of range -- -- Testing building cubes from float8 values From b7b8cc0cfcf1c956b752f3e25894f9ad607583b7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 27 Sep 2016 13:22:39 -0400 Subject: [PATCH 224/871] Redesign parallel dump/restore's wait-for-workers logic. The ListenToWorkers/ReapWorkerStatus APIs were messy and hard to use. Instead, make DispatchJobForTocEntry register a callback function that will take care of state cleanup, doing whatever had been done by the caller of ReapWorkerStatus in the old design. (This callback is essentially just the old mark_work_done function in the restore case, and a trivial test for worker failure in the dump case.) Then we can have ListenToWorkers call the callback immediately on receipt of a status message, and return the worker to WRKR_IDLE state; so the WRKR_FINISHED state goes away. This allows us to design a unified wait-for-worker-messages loop: WaitForWorkers replaces EnsureIdleWorker and EnsureWorkersFinished as well as the mess in restore_toc_entries_parallel. Also, we no longer need the fragile API spec that the caller of DispatchJobForTocEntry is responsible for ensuring there's an idle worker, since DispatchJobForTocEntry can just wait until there is one. In passing, I got rid of the ParallelArgs struct, which was a net negative in terms of notational verboseness, and didn't seem to be providing any noticeable amount of abstraction either. Tom Lane, reviewed by Kevin Grittner Discussion: <1188.1464544443@sss.pgh.pa.us> --- src/bin/pg_dump/parallel.c | 228 ++++++++++++-------------- src/bin/pg_dump/parallel.h | 72 +++++--- src/bin/pg_dump/pg_backup_archiver.c | 141 ++++++++-------- src/bin/pg_dump/pg_backup_archiver.h | 3 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 6 +- 6 files changed, 224 insertions(+), 232 deletions(-) diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index bfd023f3e1..634b4444f9 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -35,9 +35,11 @@ * the required action (dump or restore) and returns a malloc'd status string. * The status string is passed back to the master where it is interpreted by * AH->MasterEndParallelItemPtr, another format-specific routine. That - * function can update state or catalog information on the master's side, + * function can update format-specific information on the master's side, * depending on the reply from the worker process. In the end it returns a - * status code, which is 0 for successful execution. + * status code, which we pass to the ParallelCompletionPtr callback function + * that was passed to DispatchJobForTocEntry(). The callback function does + * state updating for the master control logic in pg_backup_archiver.c. * * Remember that we have forked off the workers only after we have read in * the catalog. That's why our worker processes can also access the catalog @@ -48,13 +50,8 @@ * In the master process, the workerStatus field for each worker has one of * the following values: * WRKR_IDLE: it's waiting for a command - * WRKR_WORKING: it's been sent a command - * WRKR_FINISHED: it's returned a result + * WRKR_WORKING: it's working on a command * WRKR_TERMINATED: process ended - * The FINISHED state indicates that the worker is idle, but we've not yet - * dealt with the status code it returned from the prior command. - * ReapWorkerStatus() extracts the unhandled command status value and sets - * the workerStatus back to WRKR_IDLE. */ #include "postgres_fe.h" @@ -79,6 +76,8 @@ #define PIPE_READ 0 #define PIPE_WRITE 1 +#define NO_SLOT (-1) /* Failure result for GetIdleWorker() */ + #ifdef WIN32 /* @@ -175,9 +174,12 @@ static void setup_cancel_handler(void); static void set_cancel_pstate(ParallelState *pstate); static void set_cancel_slot_archive(ParallelSlot *slot, ArchiveHandle *AH); static void RunWorker(ArchiveHandle *AH, ParallelSlot *slot); +static int GetIdleWorker(ParallelState *pstate); static bool HasEveryWorkerTerminated(ParallelState *pstate); static void lockTableForWorker(ArchiveHandle *AH, TocEntry *te); static void WaitForCommands(ArchiveHandle *AH, int pipefd[2]); +static bool ListenToWorkers(ArchiveHandle *AH, ParallelState *pstate, + bool do_wait); static char *getMessageFromMaster(int pipefd[2]); static void sendMessageToMaster(int pipefd[2], const char *str); static int select_loop(int maxFd, fd_set *workerset); @@ -349,8 +351,8 @@ archive_close_connection(int code, void *arg) * fail to detect it because there would be no EOF condition on * the other end of the pipe.) */ - if (slot->args->AH) - DisconnectDatabase(&(slot->args->AH->public)); + if (slot->AH) + DisconnectDatabase(&(slot->AH->public)); #ifdef WIN32 closesocket(slot->pipeRevRead); @@ -407,7 +409,7 @@ ShutdownWorkersHard(ParallelState *pstate) EnterCriticalSection(&signal_info_lock); for (i = 0; i < pstate->numWorkers; i++) { - ArchiveHandle *AH = pstate->parallelSlot[i].args->AH; + ArchiveHandle *AH = pstate->parallelSlot[i].AH; char errbuf[1]; if (AH != NULL && AH->connCancel != NULL) @@ -634,7 +636,7 @@ consoleHandler(DWORD dwCtrlType) for (i = 0; i < signal_info.pstate->numWorkers; i++) { ParallelSlot *slot = &(signal_info.pstate->parallelSlot[i]); - ArchiveHandle *AH = slot->args->AH; + ArchiveHandle *AH = slot->AH; HANDLE hThread = (HANDLE) slot->hThread; /* @@ -789,7 +791,7 @@ set_cancel_slot_archive(ParallelSlot *slot, ArchiveHandle *AH) EnterCriticalSection(&signal_info_lock); #endif - slot->args->AH = AH; + slot->AH = AH; #ifdef WIN32 LeaveCriticalSection(&signal_info_lock); @@ -935,9 +937,10 @@ ParallelBackupStart(ArchiveHandle *AH) strerror(errno)); slot->workerStatus = WRKR_IDLE; - slot->args = (ParallelArgs *) pg_malloc(sizeof(ParallelArgs)); - slot->args->AH = NULL; - slot->args->te = NULL; + slot->AH = NULL; + slot->te = NULL; + slot->callback = NULL; + slot->callback_data = NULL; /* master's ends of the pipes */ slot->pipeRead = pipeWM[PIPE_READ]; @@ -1071,20 +1074,28 @@ ParallelBackupEnd(ArchiveHandle *AH, ParallelState *pstate) } /* - * Dispatch a job to some free worker (caller must ensure there is one!) + * Dispatch a job to some free worker. * * te is the TocEntry to be processed, act is the action to be taken on it. + * callback is the function to call on completion of the job. + * + * If no worker is currently available, this will block, and previously + * registered callback functions may be called. */ void -DispatchJobForTocEntry(ArchiveHandle *AH, ParallelState *pstate, TocEntry *te, - T_Action act) +DispatchJobForTocEntry(ArchiveHandle *AH, + ParallelState *pstate, + TocEntry *te, + T_Action act, + ParallelCompletionPtr callback, + void *callback_data) { int worker; char *arg; - /* our caller makes sure that at least one worker is idle */ - worker = GetIdleWorker(pstate); - Assert(worker != NO_SLOT); + /* Get a worker, waiting if none are idle */ + while ((worker = GetIdleWorker(pstate)) == NO_SLOT) + WaitForWorkers(AH, pstate, WFW_ONE_IDLE); /* Construct and send command string */ arg = (AH->MasterStartParallelItemPtr) (AH, te, act); @@ -1095,14 +1106,16 @@ DispatchJobForTocEntry(ArchiveHandle *AH, ParallelState *pstate, TocEntry *te, /* Remember worker is busy, and which TocEntry it's working on */ pstate->parallelSlot[worker].workerStatus = WRKR_WORKING; - pstate->parallelSlot[worker].args->te = te; + pstate->parallelSlot[worker].te = te; + pstate->parallelSlot[worker].callback = callback; + pstate->parallelSlot[worker].callback_data = callback_data; } /* * Find an idle worker and return its slot number. * Return NO_SLOT if none are idle. */ -int +static int GetIdleWorker(ParallelState *pstate) { int i; @@ -1274,17 +1287,16 @@ WaitForCommands(ArchiveHandle *AH, int pipefd[2]) * immediately if there is none available. * * When we get a status message, we let MasterEndParallelItemPtr process it, - * then save the resulting status code and switch the worker's state to - * WRKR_FINISHED. Later, caller must call ReapWorkerStatus() to verify - * that the status was "OK" and push the worker back to IDLE state. + * then pass the resulting status code to the callback function that was + * specified to DispatchJobForTocEntry, then reset the worker status to IDLE. * - * XXX Rube Goldberg would be proud of this API, but no one else should be. + * Returns true if we collected a status message, else false. * * XXX is it worth checking for more than one status message per call? * It seems somewhat unlikely that multiple workers would finish at exactly * the same time. */ -void +static bool ListenToWorkers(ArchiveHandle *AH, ParallelState *pstate, bool do_wait) { int worker; @@ -1298,34 +1310,39 @@ ListenToWorkers(ArchiveHandle *AH, ParallelState *pstate, bool do_wait) /* If do_wait is true, we must have detected EOF on some socket */ if (do_wait) exit_horribly(modulename, "a worker process died unexpectedly\n"); - return; + return false; } /* Process it and update our idea of the worker's status */ if (messageStartsWith(msg, "OK ")) { - TocEntry *te = pstate->parallelSlot[worker].args->te; + ParallelSlot *slot = &pstate->parallelSlot[worker]; + TocEntry *te = slot->te; char *statusString; + int status; if (messageStartsWith(msg, "OK RESTORE ")) { statusString = msg + strlen("OK RESTORE "); - pstate->parallelSlot[worker].status = + status = (AH->MasterEndParallelItemPtr) (AH, te, statusString, ACT_RESTORE); + slot->callback(AH, te, status, slot->callback_data); } else if (messageStartsWith(msg, "OK DUMP ")) { statusString = msg + strlen("OK DUMP "); - pstate->parallelSlot[worker].status = + status = (AH->MasterEndParallelItemPtr) (AH, te, statusString, ACT_DUMP); + slot->callback(AH, te, status, slot->callback_data); } else exit_horribly(modulename, "invalid message received from worker: \"%s\"\n", msg); - pstate->parallelSlot[worker].workerStatus = WRKR_FINISHED; + slot->workerStatus = WRKR_IDLE; + slot->te = NULL; } else exit_horribly(modulename, @@ -1334,110 +1351,79 @@ ListenToWorkers(ArchiveHandle *AH, ParallelState *pstate, bool do_wait) /* Free the string returned from getMessageFromWorker */ free(msg); -} - -/* - * Check to see if any worker is in WRKR_FINISHED state. If so, - * return its command status code into *status, reset it to IDLE state, - * and return its slot number. Otherwise return NO_SLOT. - * - * This function is executed in the master process. - */ -int -ReapWorkerStatus(ParallelState *pstate, int *status) -{ - int i; - for (i = 0; i < pstate->numWorkers; i++) - { - if (pstate->parallelSlot[i].workerStatus == WRKR_FINISHED) - { - *status = pstate->parallelSlot[i].status; - pstate->parallelSlot[i].status = 0; - pstate->parallelSlot[i].workerStatus = WRKR_IDLE; - return i; - } - } - return NO_SLOT; + return true; } /* - * Wait, if necessary, until we have at least one idle worker. - * Reap worker status as necessary to move FINISHED workers to IDLE state. + * Check for status results from workers, waiting if necessary. * - * We assume that no extra processing is required when reaping a finished - * command, except for checking that the status was OK (zero). - * Caution: that assumption means that this function can only be used in - * parallel dump, not parallel restore, because the latter has a more - * complex set of rules about handling status. + * Available wait modes are: + * WFW_NO_WAIT: reap any available status, but don't block + * WFW_GOT_STATUS: wait for at least one more worker to finish + * WFW_ONE_IDLE: wait for at least one worker to be idle + * WFW_ALL_IDLE: wait for all workers to be idle + * + * Any received results are passed to MasterEndParallelItemPtr and then + * to the callback specified to DispatchJobForTocEntry. * * This function is executed in the master process. */ void -EnsureIdleWorker(ArchiveHandle *AH, ParallelState *pstate) +WaitForWorkers(ArchiveHandle *AH, ParallelState *pstate, WFW_WaitOption mode) { - int ret_worker; - int work_status; + bool do_wait = false; - for (;;) + /* + * In GOT_STATUS mode, always block waiting for a message, since we can't + * return till we get something. In other modes, we don't block the first + * time through the loop. + */ + if (mode == WFW_GOT_STATUS) { - int nTerm = 0; - - while ((ret_worker = ReapWorkerStatus(pstate, &work_status)) != NO_SLOT) - { - if (work_status != 0) - exit_horribly(modulename, "error processing a parallel work item\n"); - - nTerm++; - } - - /* - * We need to make sure that we have an idle worker before dispatching - * the next item. If nTerm > 0 we already have that (quick check). - */ - if (nTerm > 0) - return; - - /* explicit check for an idle worker */ - if (GetIdleWorker(pstate) != NO_SLOT) - return; + /* Assert that caller knows what it's doing */ + Assert(!IsEveryWorkerIdle(pstate)); + do_wait = true; + } + for (;;) + { /* - * If we have no idle worker, read the result of one or more workers - * and loop the loop to call ReapWorkerStatus() on them + * Check for status messages, even if we don't need to block. We do + * not try very hard to reap all available messages, though, since + * there's unlikely to be more than one. */ - ListenToWorkers(AH, pstate, true); - } -} - -/* - * Wait for all workers to be idle. - * Reap worker status as necessary to move FINISHED workers to IDLE state. - * - * We assume that no extra processing is required when reaping a finished - * command, except for checking that the status was OK (zero). - * Caution: that assumption means that this function can only be used in - * parallel dump, not parallel restore, because the latter has a more - * complex set of rules about handling status. - * - * This function is executed in the master process. - */ -void -EnsureWorkersFinished(ArchiveHandle *AH, ParallelState *pstate) -{ - int work_status; + if (ListenToWorkers(AH, pstate, do_wait)) + { + /* + * If we got a message, we are done by definition for GOT_STATUS + * mode, and we can also be certain that there's at least one idle + * worker. So we're done in all but ALL_IDLE mode. + */ + if (mode != WFW_ALL_IDLE) + return; + } - if (!pstate || pstate->numWorkers == 1) - return; + /* Check whether we must wait for new status messages */ + switch (mode) + { + case WFW_NO_WAIT: + return; /* never wait */ + case WFW_GOT_STATUS: + Assert(false); /* can't get here, because we waited */ + break; + case WFW_ONE_IDLE: + if (GetIdleWorker(pstate) != NO_SLOT) + return; + break; + case WFW_ALL_IDLE: + if (IsEveryWorkerIdle(pstate)) + return; + break; + } - /* Waiting for the remaining worker processes to finish */ - while (!IsEveryWorkerIdle(pstate)) - { - if (ReapWorkerStatus(pstate, &work_status) == NO_SLOT) - ListenToWorkers(AH, pstate, true); - else if (work_status != 0) - exit_horribly(modulename, - "error processing a parallel work item\n"); + /* Loop back, and this time wait for something to happen */ + do_wait = true; } } diff --git a/src/bin/pg_dump/parallel.h b/src/bin/pg_dump/parallel.h index 21739ca87c..8ee629b106 100644 --- a/src/bin/pg_dump/parallel.h +++ b/src/bin/pg_dump/parallel.h @@ -2,14 +2,11 @@ * * parallel.h * - * Parallel support header file for the pg_dump archiver + * Parallel support for pg_dump and pg_restore * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * The author is not responsible for loss or damages that may - * result from its use. - * * IDENTIFICATION * src/bin/pg_dump/parallel.h * @@ -21,31 +18,53 @@ #include "pg_backup_archiver.h" +/* Function to call in master process on completion of a worker task */ +typedef void (*ParallelCompletionPtr) (ArchiveHandle *AH, + TocEntry *te, + int status, + void *callback_data); + +/* Wait options for WaitForWorkers */ +typedef enum +{ + WFW_NO_WAIT, + WFW_GOT_STATUS, + WFW_ONE_IDLE, + WFW_ALL_IDLE +} WFW_WaitOption; + +/* Worker process statuses */ typedef enum { - WRKR_TERMINATED = 0, WRKR_IDLE, WRKR_WORKING, - WRKR_FINISHED + WRKR_TERMINATED } T_WorkerStatus; -/* Arguments needed for a worker process */ -typedef struct ParallelArgs -{ - ArchiveHandle *AH; - TocEntry *te; -} ParallelArgs; - -/* State for each parallel activity slot */ +/* + * Per-parallel-worker state of parallel.c. + * + * Much of this is valid only in the master process (or, on Windows, should + * be touched only by the master thread). But the AH field should be touched + * only by workers. The pipe descriptors are valid everywhere. + */ typedef struct ParallelSlot { - ParallelArgs *args; - T_WorkerStatus workerStatus; - int status; + T_WorkerStatus workerStatus; /* see enum above */ + + /* These fields are valid if workerStatus == WRKR_WORKING: */ + TocEntry *te; /* item being worked on */ + ParallelCompletionPtr callback; /* function to call on completion */ + void *callback_data; /* passthru data for it */ + + ArchiveHandle *AH; /* Archive data worker is using */ + int pipeRead; /* master's end of the pipes */ int pipeWrite; int pipeRevRead; /* child's end of the pipes */ int pipeRevWrite; + + /* Child process/thread identity info: */ #ifdef WIN32 uintptr_t hThread; unsigned int threadId; @@ -54,12 +73,11 @@ typedef struct ParallelSlot #endif } ParallelSlot; -#define NO_SLOT (-1) - +/* Overall state for parallel.c */ typedef struct ParallelState { - int numWorkers; - ParallelSlot *parallelSlot; + int numWorkers; /* allowed number of workers */ + ParallelSlot *parallelSlot; /* array of numWorkers slots */ } ParallelState; #ifdef WIN32 @@ -69,17 +87,17 @@ extern DWORD mainThreadId; extern void init_parallel_dump_utils(void); -extern int GetIdleWorker(ParallelState *pstate); extern bool IsEveryWorkerIdle(ParallelState *pstate); -extern void ListenToWorkers(ArchiveHandle *AH, ParallelState *pstate, bool do_wait); -extern int ReapWorkerStatus(ParallelState *pstate, int *status); -extern void EnsureIdleWorker(ArchiveHandle *AH, ParallelState *pstate); -extern void EnsureWorkersFinished(ArchiveHandle *AH, ParallelState *pstate); +extern void WaitForWorkers(ArchiveHandle *AH, ParallelState *pstate, + WFW_WaitOption mode); extern ParallelState *ParallelBackupStart(ArchiveHandle *AH); extern void DispatchJobForTocEntry(ArchiveHandle *AH, ParallelState *pstate, - TocEntry *te, T_Action act); + TocEntry *te, + T_Action act, + ParallelCompletionPtr callback, + void *callback_data); extern void ParallelBackupEnd(ArchiveHandle *AH, ParallelState *pstate); extern void set_archive_cancel_info(ArchiveHandle *AH, PGconn *conn); diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index a69b06f6d7..e19c24aec9 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -97,9 +97,14 @@ static void par_list_remove(TocEntry *te); static TocEntry *get_next_work_item(ArchiveHandle *AH, TocEntry *ready_list, ParallelState *pstate); -static void mark_work_done(ArchiveHandle *AH, TocEntry *ready_list, - int worker, int status, - ParallelState *pstate); +static void mark_dump_job_done(ArchiveHandle *AH, + TocEntry *te, + int status, + void *callback_data); +static void mark_restore_job_done(ArchiveHandle *AH, + TocEntry *te, + int status, + void *callback_data); static void fix_dependencies(ArchiveHandle *AH); static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2); static void repoint_table_dependencies(ArchiveHandle *AH); @@ -2355,8 +2360,8 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate) * If we are in a parallel backup, then we are always the master * process. Dispatch each data-transfer job to a worker. */ - EnsureIdleWorker(AH, pstate); - DispatchJobForTocEntry(AH, pstate, te, ACT_DUMP); + DispatchJobForTocEntry(AH, pstate, te, ACT_DUMP, + mark_dump_job_done, NULL); } else WriteDataChunksForTocEntry(AH, te); @@ -2365,9 +2370,32 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate) /* * If parallel, wait for workers to finish. */ - EnsureWorkersFinished(AH, pstate); + if (pstate && pstate->numWorkers > 1) + WaitForWorkers(AH, pstate, WFW_ALL_IDLE); } + +/* + * Callback function that's invoked in the master process after a step has + * been parallel dumped. + * + * We don't need to do anything except check for worker failure. + */ +static void +mark_dump_job_done(ArchiveHandle *AH, + TocEntry *te, + int status, + void *callback_data) +{ + ahlog(AH, 1, "finished item %d %s %s\n", + te->dumpId, te->desc, te->tag); + + if (status != 0) + exit_horribly(modulename, "worker process failed: exit code %d\n", + status); +} + + void WriteDataChunksForTocEntry(ArchiveHandle *AH, TocEntry *te) { @@ -2751,9 +2779,9 @@ _tocEntryRequired(TocEntry *te, teSection curSection, RestoreOptions *ropt) return 0; } - if (ropt->schemaExcludeNames.head != NULL - && te->namespace - && simple_string_list_member(&ropt->schemaExcludeNames, te->namespace)) + if (ropt->schemaExcludeNames.head != NULL && + te->namespace && + simple_string_list_member(&ropt->schemaExcludeNames, te->namespace)) return 0; if (ropt->selTypes) @@ -3769,11 +3797,9 @@ static void restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate, TocEntry *pending_list) { - int work_status; bool skipped_some; TocEntry ready_list; TocEntry *next_work_item; - int ret_child; ahlog(AH, 2, "entering restore_toc_entries_parallel\n"); @@ -3850,54 +3876,29 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate, par_list_remove(next_work_item); - DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE); + DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE, + mark_restore_job_done, &ready_list); } else { /* at least one child is working and we have nothing ready. */ } - for (;;) - { - int nTerm = 0; - - /* - * In order to reduce dependencies as soon as possible and - * especially to reap the status of workers who are working on - * items that pending items depend on, we do a non-blocking check - * for ended workers first. - * - * However, if we do not have any other work items currently that - * workers can work on, we do not busy-loop here but instead - * really wait for at least one worker to terminate. Hence we call - * ListenToWorkers(..., ..., do_wait = true) in this case. - */ - ListenToWorkers(AH, pstate, !next_work_item); - - while ((ret_child = ReapWorkerStatus(pstate, &work_status)) != NO_SLOT) - { - nTerm++; - mark_work_done(AH, &ready_list, ret_child, work_status, pstate); - } - - /* - * We need to make sure that we have an idle worker before - * re-running the loop. If nTerm > 0 we already have that (quick - * check). - */ - if (nTerm > 0) - break; - - /* if nobody terminated, explicitly check for an idle worker */ - if (GetIdleWorker(pstate) != NO_SLOT) - break; - - /* - * If we have no idle worker, read the result of one or more - * workers and loop the loop to call ReapWorkerStatus() on them. - */ - ListenToWorkers(AH, pstate, true); - } + /* + * Before dispatching another job, check to see if anything has + * finished. We should check every time through the loop so as to + * reduce dependencies as soon as possible. If we were unable to + * dispatch any job this time through, wait until some worker finishes + * (and, hopefully, unblocks some pending item). If we did dispatch + * something, continue as soon as there's at least one idle worker. + * Note that in either case, there's guaranteed to be at least one + * idle worker when we return to the top of the loop. This ensures we + * won't block inside DispatchJobForTocEntry, which would be + * undesirable: we'd rather postpone dispatching until we see what's + * been unblocked by finished jobs. + */ + WaitForWorkers(AH, pstate, + next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS); } ahlog(AH, 1, "finished main parallel loop\n"); @@ -4025,9 +4026,11 @@ get_next_work_item(ArchiveHandle *AH, TocEntry *ready_list, int count = 0; for (k = 0; k < pstate->numWorkers; k++) - if (pstate->parallelSlot[k].args->te != NULL && - pstate->parallelSlot[k].args->te->section == SECTION_DATA) + { + if (pstate->parallelSlot[k].workerStatus == WRKR_WORKING && + pstate->parallelSlot[k].te->section == SECTION_DATA) count++; + } if (pstate->numWorkers == 0 || count * 4 < pstate->numWorkers) pref_non_data = false; } @@ -4044,13 +4047,13 @@ get_next_work_item(ArchiveHandle *AH, TocEntry *ready_list, * that a currently running item also needs lock on, or vice versa. If * so, we don't want to schedule them together. */ - for (i = 0; i < pstate->numWorkers && !conflicts; i++) + for (i = 0; i < pstate->numWorkers; i++) { TocEntry *running_te; if (pstate->parallelSlot[i].workerStatus != WRKR_WORKING) continue; - running_te = pstate->parallelSlot[i].args->te; + running_te = pstate->parallelSlot[i].te; if (has_lock_conflicts(te, running_te) || has_lock_conflicts(running_te, te)) @@ -4091,10 +4094,8 @@ get_next_work_item(ArchiveHandle *AH, TocEntry *ready_list, * our work is finished, the master process will assign us a new work item. */ int -parallel_restore(ParallelArgs *args) +parallel_restore(ArchiveHandle *AH, TocEntry *te) { - ArchiveHandle *AH = args->AH; - TocEntry *te = args->te; int status; Assert(AH->connection != NULL); @@ -4110,22 +4111,18 @@ parallel_restore(ParallelArgs *args) /* - * Housekeeping to be done after a step has been parallel restored. + * Callback function that's invoked in the master process after a step has + * been parallel restored. * - * Clear the appropriate slot, free all the extra memory we allocated, - * update status, and reduce the dependency count of any dependent items. + * Update status and reduce the dependency count of any dependent items. */ static void -mark_work_done(ArchiveHandle *AH, TocEntry *ready_list, - int worker, int status, - ParallelState *pstate) +mark_restore_job_done(ArchiveHandle *AH, + TocEntry *te, + int status, + void *callback_data) { - TocEntry *te = NULL; - - te = pstate->parallelSlot[worker].args->te; - - if (te == NULL) - exit_horribly(modulename, "could not find slot of finished worker\n"); + TocEntry *ready_list = (TocEntry *) callback_data; ahlog(AH, 1, "finished item %d %s %s\n", te->dumpId, te->desc, te->tag); diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index 0376f2bff7..123aa5dc84 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -111,7 +111,6 @@ typedef z_stream *z_streamp; typedef struct _archiveHandle ArchiveHandle; typedef struct _tocEntry TocEntry; -struct ParallelArgs; struct ParallelState; #define READ_ERROR_EXIT(fd) \ @@ -375,7 +374,7 @@ struct _tocEntry int nLockDeps; /* number of such dependencies */ }; -extern int parallel_restore(struct ParallelArgs *args); +extern int parallel_restore(ArchiveHandle *AH, TocEntry *te); extern void on_exit_close_archive(Archive *AHX); extern void warn_or_exit_horribly(ArchiveHandle *AH, const char *modulename, const char *fmt,...) pg_attribute_printf(3, 4); diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 66329dc90c..c4f487a7ca 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -820,13 +820,9 @@ _WorkerJobRestoreCustom(ArchiveHandle *AH, TocEntry *te) */ const int buflen = 64; char *buf = (char *) pg_malloc(buflen); - ParallelArgs pargs; int status; - pargs.AH = AH; - pargs.te = te; - - status = parallel_restore(&pargs); + status = parallel_restore(AH, te); snprintf(buf, buflen, "OK RESTORE %d %d %d", te->dumpId, status, status == WORKER_IGNORED_ERRORS ? AH->public.n_errors : 0); diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index e52f12258a..8071259acb 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -826,13 +826,9 @@ _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te) */ const int buflen = 64; char *buf = (char *) pg_malloc(buflen); - ParallelArgs pargs; int status; - pargs.AH = AH; - pargs.te = te; - - status = parallel_restore(&pargs); + status = parallel_restore(AH, te); snprintf(buf, buflen, "OK RESTORE %d %d %d", te->dumpId, status, status == WORKER_IGNORED_ERRORS ? AH->public.n_errors : 0); From fb03d08a89e81a68585f17fd8e7f21c618f4e851 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 27 Sep 2016 13:56:04 -0400 Subject: [PATCH 225/871] Rationalize parallel dump/restore's handling of worker cmd/status messages. The existing APIs for creating and parsing command and status messages are rather messy; for example, archive-format modules have to provide code for constructing command messages, which is entirely pointless since the code to read them is hard-wired in WaitForCommands() and hence no format-specific variation is actually possible. But there's little foreseeable reason to need format-specific variation anyway. The situation for status messages is no better; at least those are both constructed and parsed by format-specific code, but said code is quite redundant since there's no actual need for format-specific variation. To add insult to injury, the first API involves returning pointers to static buffers, which is bad, while the second involves returning pointers to malloc'd strings, which is safer but randomly inconsistent. Hence, get rid of the MasterStartParallelItem and MasterEndParallelItem APIs, and instead write centralized functions that construct and parse command and status messages. If we ever do need more flexibility, these functions can be the standard implementations of format-specific callback methods, but that's a long way off if it ever happens. Tom Lane, reviewed by Kevin Grittner Discussion: <17340.1464465717@sss.pgh.pa.us> --- src/bin/pg_dump/parallel.c | 232 ++++++++++++++++---------- src/bin/pg_dump/pg_backup_archiver.h | 11 +- src/bin/pg_dump/pg_backup_custom.c | 75 +-------- src/bin/pg_dump/pg_backup_directory.c | 115 ++----------- src/bin/pg_dump/pg_backup_tar.c | 3 - 5 files changed, 162 insertions(+), 274 deletions(-) diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index 634b4444f9..0e2bfa106a 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -20,32 +20,25 @@ * the desired number of worker processes, which each enter WaitForCommands(). * * The master process dispatches an individual work item to one of the worker - * processes in DispatchJobForTocEntry(). That calls - * AH->MasterStartParallelItemPtr, a routine of the output format. This - * function's arguments are the parents archive handle AH (containing the full - * catalog information), the TocEntry that the worker should work on and a - * T_Action value indicating whether this is a backup or a restore task. The - * function simply converts the TocEntry assignment into a command string that - * is then sent over to the worker process. In the simplest case that would be - * something like "DUMP 1234", with 1234 being the TocEntry id. - * + * processes in DispatchJobForTocEntry(). We send a command string such as + * "DUMP 1234" or "RESTORE 1234", where 1234 is the TocEntry ID. * The worker process receives and decodes the command and passes it to the * routine pointed to by AH->WorkerJobDumpPtr or AH->WorkerJobRestorePtr, * which are routines of the current archive format. That routine performs - * the required action (dump or restore) and returns a malloc'd status string. - * The status string is passed back to the master where it is interpreted by - * AH->MasterEndParallelItemPtr, another format-specific routine. That - * function can update format-specific information on the master's side, - * depending on the reply from the worker process. In the end it returns a - * status code, which we pass to the ParallelCompletionPtr callback function - * that was passed to DispatchJobForTocEntry(). The callback function does - * state updating for the master control logic in pg_backup_archiver.c. + * the required action (dump or restore) and returns an integer status code. + * This is passed back to the master where we pass it to the + * ParallelCompletionPtr callback function that was passed to + * DispatchJobForTocEntry(). The callback function does state updating + * for the master control logic in pg_backup_archiver.c. * - * Remember that we have forked off the workers only after we have read in - * the catalog. That's why our worker processes can also access the catalog - * information. (In the Windows case, the workers are threads in the same - * process. To avoid problems, they work with cloned copies of the Archive - * data structure; see RunWorker().) + * In principle additional archive-format-specific information might be needed + * in commands or worker status responses, but so far that hasn't proved + * necessary, since workers have full copies of the ArchiveHandle/TocEntry + * data structures. Remember that we have forked off the workers only after + * we have read in the catalog. That's why our worker processes can also + * access the catalog information. (In the Windows case, the workers are + * threads in the same process. To avoid problems, they work with cloned + * copies of the Archive data structure; see RunWorker().) * * In the master process, the workerStatus field for each worker has one of * the following values: @@ -1073,6 +1066,110 @@ ParallelBackupEnd(ArchiveHandle *AH, ParallelState *pstate) free(pstate); } +/* + * These next four functions handle construction and parsing of the command + * strings and response strings for parallel workers. + * + * Currently, these can be the same regardless of which archive format we are + * processing. In future, we might want to let format modules override these + * functions to add format-specific data to a command or response. + */ + +/* + * buildWorkerCommand: format a command string to send to a worker. + * + * The string is built in the caller-supplied buffer of size buflen. + */ +static void +buildWorkerCommand(ArchiveHandle *AH, TocEntry *te, T_Action act, + char *buf, int buflen) +{ + if (act == ACT_DUMP) + snprintf(buf, buflen, "DUMP %d", te->dumpId); + else if (act == ACT_RESTORE) + snprintf(buf, buflen, "RESTORE %d", te->dumpId); + else + Assert(false); +} + +/* + * parseWorkerCommand: interpret a command string in a worker. + */ +static void +parseWorkerCommand(ArchiveHandle *AH, TocEntry **te, T_Action *act, + const char *msg) +{ + DumpId dumpId; + int nBytes; + + if (messageStartsWith(msg, "DUMP ")) + { + *act = ACT_DUMP; + sscanf(msg, "DUMP %d%n", &dumpId, &nBytes); + Assert(nBytes == strlen(msg)); + *te = getTocEntryByDumpId(AH, dumpId); + Assert(*te != NULL); + } + else if (messageStartsWith(msg, "RESTORE ")) + { + *act = ACT_RESTORE; + sscanf(msg, "RESTORE %d%n", &dumpId, &nBytes); + Assert(nBytes == strlen(msg)); + *te = getTocEntryByDumpId(AH, dumpId); + Assert(*te != NULL); + } + else + exit_horribly(modulename, + "unrecognized command received from master: \"%s\"\n", + msg); +} + +/* + * buildWorkerResponse: format a response string to send to the master. + * + * The string is built in the caller-supplied buffer of size buflen. + */ +static void +buildWorkerResponse(ArchiveHandle *AH, TocEntry *te, T_Action act, int status, + char *buf, int buflen) +{ + snprintf(buf, buflen, "OK %d %d %d", + te->dumpId, + status, + status == WORKER_IGNORED_ERRORS ? AH->public.n_errors : 0); +} + +/* + * parseWorkerResponse: parse the status message returned by a worker. + * + * Returns the integer status code, and may update fields of AH and/or te. + */ +static int +parseWorkerResponse(ArchiveHandle *AH, TocEntry *te, + const char *msg) +{ + DumpId dumpId; + int nBytes, + n_errors; + int status = 0; + + if (messageStartsWith(msg, "OK ")) + { + sscanf(msg, "OK %d %d %d%n", &dumpId, &status, &n_errors, &nBytes); + + Assert(dumpId == te->dumpId); + Assert(nBytes == strlen(msg)); + + AH->public.n_errors += n_errors; + } + else + exit_horribly(modulename, + "invalid message received from worker: \"%s\"\n", + msg); + + return status; +} + /* * Dispatch a job to some free worker. * @@ -1091,18 +1188,16 @@ DispatchJobForTocEntry(ArchiveHandle *AH, void *callback_data) { int worker; - char *arg; + char buf[256]; /* Get a worker, waiting if none are idle */ while ((worker = GetIdleWorker(pstate)) == NO_SLOT) WaitForWorkers(AH, pstate, WFW_ONE_IDLE); /* Construct and send command string */ - arg = (AH->MasterStartParallelItemPtr) (AH, te, act); - - sendMessageToWorker(pstate, worker, arg); + buildWorkerCommand(AH, te, act, buf, sizeof(buf)); - /* XXX aren't we leaking string here? (no, because it's static. Ick.) */ + sendMessageToWorker(pstate, worker, buf); /* Remember worker is busy, and which TocEntry it's working on */ pstate->parallelSlot[worker].workerStatus = WRKR_WORKING; @@ -1220,10 +1315,10 @@ static void WaitForCommands(ArchiveHandle *AH, int pipefd[2]) { char *command; - DumpId dumpId; - int nBytes; - char *str; TocEntry *te; + T_Action act; + int status = 0; + char buf[256]; for (;;) { @@ -1233,47 +1328,29 @@ WaitForCommands(ArchiveHandle *AH, int pipefd[2]) return; } - if (messageStartsWith(command, "DUMP ")) - { - /* Decode the command */ - sscanf(command + strlen("DUMP "), "%d%n", &dumpId, &nBytes); - Assert(nBytes == strlen(command) - strlen("DUMP ")); - te = getTocEntryByDumpId(AH, dumpId); - Assert(te != NULL); + /* Decode the command */ + parseWorkerCommand(AH, &te, &act, command); + if (act == ACT_DUMP) + { /* Acquire lock on this table within the worker's session */ lockTableForWorker(AH, te); /* Perform the dump command */ - str = (AH->WorkerJobDumpPtr) (AH, te); - - /* Return status to master */ - sendMessageToMaster(pipefd, str); - - /* we are responsible for freeing the status string */ - free(str); + status = (AH->WorkerJobDumpPtr) (AH, te); } - else if (messageStartsWith(command, "RESTORE ")) + else if (act == ACT_RESTORE) { - /* Decode the command */ - sscanf(command + strlen("RESTORE "), "%d%n", &dumpId, &nBytes); - Assert(nBytes == strlen(command) - strlen("RESTORE ")); - te = getTocEntryByDumpId(AH, dumpId); - Assert(te != NULL); - /* Perform the restore command */ - str = (AH->WorkerJobRestorePtr) (AH, te); - - /* Return status to master */ - sendMessageToMaster(pipefd, str); - - /* we are responsible for freeing the status string */ - free(str); + status = (AH->WorkerJobRestorePtr) (AH, te); } else - exit_horribly(modulename, - "unrecognized command received from master: \"%s\"\n", - command); + Assert(false); + + /* Return status to master */ + buildWorkerResponse(AH, te, act, status, buf, sizeof(buf)); + + sendMessageToMaster(pipefd, buf); /* command was pg_malloc'd and we are responsible for free()ing it. */ free(command); @@ -1286,9 +1363,9 @@ WaitForCommands(ArchiveHandle *AH, int pipefd[2]) * If do_wait is true, wait to get a status message; otherwise, just return * immediately if there is none available. * - * When we get a status message, we let MasterEndParallelItemPtr process it, - * then pass the resulting status code to the callback function that was - * specified to DispatchJobForTocEntry, then reset the worker status to IDLE. + * When we get a status message, we pass the status code to the callback + * function that was specified to DispatchJobForTocEntry, then reset the + * worker status to IDLE. * * Returns true if we collected a status message, else false. * @@ -1318,29 +1395,10 @@ ListenToWorkers(ArchiveHandle *AH, ParallelState *pstate, bool do_wait) { ParallelSlot *slot = &pstate->parallelSlot[worker]; TocEntry *te = slot->te; - char *statusString; int status; - if (messageStartsWith(msg, "OK RESTORE ")) - { - statusString = msg + strlen("OK RESTORE "); - status = - (AH->MasterEndParallelItemPtr) - (AH, te, statusString, ACT_RESTORE); - slot->callback(AH, te, status, slot->callback_data); - } - else if (messageStartsWith(msg, "OK DUMP ")) - { - statusString = msg + strlen("OK DUMP "); - status = - (AH->MasterEndParallelItemPtr) - (AH, te, statusString, ACT_DUMP); - slot->callback(AH, te, status, slot->callback_data); - } - else - exit_horribly(modulename, - "invalid message received from worker: \"%s\"\n", - msg); + status = parseWorkerResponse(AH, te, msg); + slot->callback(AH, te, status, slot->callback_data); slot->workerStatus = WRKR_IDLE; slot->te = NULL; } @@ -1364,8 +1422,8 @@ ListenToWorkers(ArchiveHandle *AH, ParallelState *pstate, bool do_wait) * WFW_ONE_IDLE: wait for at least one worker to be idle * WFW_ALL_IDLE: wait for all workers to be idle * - * Any received results are passed to MasterEndParallelItemPtr and then - * to the callback specified to DispatchJobForTocEntry. + * Any received results are passed to the callback specified to + * DispatchJobForTocEntry. * * This function is executed in the master process. */ diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index 123aa5dc84..97d34a5297 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -161,12 +161,8 @@ typedef void (*PrintTocDataPtr) (ArchiveHandle *AH, TocEntry *te); typedef void (*ClonePtr) (ArchiveHandle *AH); typedef void (*DeClonePtr) (ArchiveHandle *AH); -typedef char *(*WorkerJobRestorePtr) (ArchiveHandle *AH, TocEntry *te); -typedef char *(*WorkerJobDumpPtr) (ArchiveHandle *AH, TocEntry *te); -typedef char *(*MasterStartParallelItemPtr) (ArchiveHandle *AH, TocEntry *te, - T_Action act); -typedef int (*MasterEndParallelItemPtr) (ArchiveHandle *AH, TocEntry *te, - const char *str, T_Action act); +typedef int (*WorkerJobDumpPtr) (ArchiveHandle *AH, TocEntry *te); +typedef int (*WorkerJobRestorePtr) (ArchiveHandle *AH, TocEntry *te); typedef size_t (*CustomOutPtr) (ArchiveHandle *AH, const void *buf, size_t len); @@ -266,9 +262,6 @@ struct _archiveHandle StartBlobPtr StartBlobPtr; EndBlobPtr EndBlobPtr; - MasterStartParallelItemPtr MasterStartParallelItemPtr; - MasterEndParallelItemPtr MasterEndParallelItemPtr; - SetupWorkerPtr SetupWorkerPtr; WorkerJobDumpPtr WorkerJobDumpPtr; WorkerJobRestorePtr WorkerJobRestorePtr; diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index c4f487a7ca..5388c08b29 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -61,9 +61,7 @@ static void _LoadBlobs(ArchiveHandle *AH, bool drop); static void _Clone(ArchiveHandle *AH); static void _DeClone(ArchiveHandle *AH); -static char *_MasterStartParallelItem(ArchiveHandle *AH, TocEntry *te, T_Action act); -static int _MasterEndParallelItem(ArchiveHandle *AH, TocEntry *te, const char *str, T_Action act); -char *_WorkerJobRestoreCustom(ArchiveHandle *AH, TocEntry *te); +static int _WorkerJobRestoreCustom(ArchiveHandle *AH, TocEntry *te); typedef struct { @@ -133,9 +131,6 @@ InitArchiveFmt_Custom(ArchiveHandle *AH) AH->ClonePtr = _Clone; AH->DeClonePtr = _DeClone; - AH->MasterStartParallelItemPtr = _MasterStartParallelItem; - AH->MasterEndParallelItemPtr = _MasterEndParallelItem; - /* no parallel dump in the custom archive, only parallel restore */ AH->WorkerJobDumpPtr = NULL; AH->WorkerJobRestorePtr = _WorkerJobRestoreCustom; @@ -808,73 +803,13 @@ _DeClone(ArchiveHandle *AH) } /* - * This function is executed in the child of a parallel backup for the - * custom format archive and dumps the actual data. - */ -char * -_WorkerJobRestoreCustom(ArchiveHandle *AH, TocEntry *te) -{ - /* - * short fixed-size string + some ID so far, this needs to be malloc'ed - * instead of static because we work with threads on windows - */ - const int buflen = 64; - char *buf = (char *) pg_malloc(buflen); - int status; - - status = parallel_restore(AH, te); - - snprintf(buf, buflen, "OK RESTORE %d %d %d", te->dumpId, status, - status == WORKER_IGNORED_ERRORS ? AH->public.n_errors : 0); - - return buf; -} - -/* - * This function is executed in the parent process. Depending on the desired - * action (dump or restore) it creates a string that is understood by the - * _WorkerJobDump /_WorkerJobRestore functions of the dump format. - */ -static char * -_MasterStartParallelItem(ArchiveHandle *AH, TocEntry *te, T_Action act) -{ - /* - * A static char is okay here, even on Windows because we call this - * function only from one process (the master). - */ - static char buf[64]; /* short fixed-size string + number */ - - /* no parallel dump in the custom archive format */ - Assert(act == ACT_RESTORE); - - snprintf(buf, sizeof(buf), "RESTORE %d", te->dumpId); - - return buf; -} - -/* - * This function is executed in the parent process. It analyzes the response of - * the _WorkerJobDump / _WorkerJobRestore functions of the dump format. + * This function is executed in the child of a parallel restore from a + * custom-format archive and restores the actual data for one TOC entry. */ static int -_MasterEndParallelItem(ArchiveHandle *AH, TocEntry *te, const char *str, T_Action act) +_WorkerJobRestoreCustom(ArchiveHandle *AH, TocEntry *te) { - DumpId dumpId; - int nBytes, - status, - n_errors; - - /* no parallel dump in the custom archive */ - Assert(act == ACT_RESTORE); - - sscanf(str, "%d %d %d%n", &dumpId, &status, &n_errors, &nBytes); - - Assert(nBytes == strlen(str)); - Assert(dumpId == te->dumpId); - - AH->public.n_errors += n_errors; - - return status; + return parallel_restore(AH, te); } /*-------------------------------------------------- diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 8071259acb..ae443717cf 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -89,11 +89,8 @@ static void _LoadBlobs(ArchiveHandle *AH); static void _Clone(ArchiveHandle *AH); static void _DeClone(ArchiveHandle *AH); -static char *_MasterStartParallelItem(ArchiveHandle *AH, TocEntry *te, T_Action act); -static int _MasterEndParallelItem(ArchiveHandle *AH, TocEntry *te, - const char *str, T_Action act); -static char *_WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te); -static char *_WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te); +static int _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te); +static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te); static void setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename); @@ -140,9 +137,6 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) AH->WorkerJobRestorePtr = _WorkerJobRestoreDirectory; AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory; - AH->MasterStartParallelItemPtr = _MasterStartParallelItem; - AH->MasterEndParallelItemPtr = _MasterEndParallelItem; - /* Set up our private context */ ctx = (lclContext *) pg_malloc0(sizeof(lclContext)); AH->formatData = (void *) ctx; @@ -754,53 +748,12 @@ _DeClone(ArchiveHandle *AH) } /* - * This function is executed in the parent process. Depending on the desired - * action (dump or restore) it creates a string that is understood by the - * _WorkerJobDump /_WorkerJobRestore functions of the dump format. - */ -static char * -_MasterStartParallelItem(ArchiveHandle *AH, TocEntry *te, T_Action act) -{ - /* - * A static char is okay here, even on Windows because we call this - * function only from one process (the master). - */ - static char buf[64]; - - if (act == ACT_DUMP) - snprintf(buf, sizeof(buf), "DUMP %d", te->dumpId); - else if (act == ACT_RESTORE) - snprintf(buf, sizeof(buf), "RESTORE %d", te->dumpId); - - return buf; -} - -/* - * This function is executed in the child of a parallel backup for the - * directory archive and dumps the actual data. - * - * We are currently returning only the DumpId so theoretically we could - * make this function returning an int (or a DumpId). However, to - * facilitate further enhancements and because sooner or later we need to - * convert this to a string and send it via a message anyway, we stick with - * char *. It is parsed on the other side by the _EndMasterParallel() - * function of the respective dump format. + * This function is executed in the child of a parallel backup for a + * directory-format archive and dumps the actual data for one TOC entry. */ -static char * +static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te) { - /* - * short fixed-size string + some ID so far, this needs to be malloc'ed - * instead of static because we work with threads on windows - */ - const int buflen = 64; - char *buf = (char *) pg_malloc(buflen); - lclTocEntry *tctx = (lclTocEntry *) te->formatData; - - /* This should never happen */ - if (!tctx) - exit_horribly(modulename, "error during backup\n"); - /* * This function returns void. We either fail and die horribly or * succeed... A failure will be detected by the parent when the child dies @@ -808,63 +761,15 @@ _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te) */ WriteDataChunksForTocEntry(AH, te); - snprintf(buf, buflen, "OK DUMP %d", te->dumpId); - - return buf; + return 0; } /* - * This function is executed in the child of a parallel backup for the - * directory archive and dumps the actual data. - */ -static char * -_WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te) -{ - /* - * short fixed-size string + some ID so far, this needs to be malloc'ed - * instead of static because we work with threads on windows - */ - const int buflen = 64; - char *buf = (char *) pg_malloc(buflen); - int status; - - status = parallel_restore(AH, te); - - snprintf(buf, buflen, "OK RESTORE %d %d %d", te->dumpId, status, - status == WORKER_IGNORED_ERRORS ? AH->public.n_errors : 0); - - return buf; -} - -/* - * This function is executed in the parent process. It analyzes the response of - * the _WorkerJobDumpDirectory/_WorkerJobRestoreDirectory functions of the - * respective dump format. + * This function is executed in the child of a parallel restore from a + * directory-format archive and restores the actual data for one TOC entry. */ static int -_MasterEndParallelItem(ArchiveHandle *AH, TocEntry *te, const char *str, T_Action act) +_WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te) { - DumpId dumpId; - int nBytes, - n_errors; - int status = 0; - - if (act == ACT_DUMP) - { - sscanf(str, "%d%n", &dumpId, &nBytes); - - Assert(dumpId == te->dumpId); - Assert(nBytes == strlen(str)); - } - else if (act == ACT_RESTORE) - { - sscanf(str, "%d %d %d%n", &dumpId, &status, &n_errors, &nBytes); - - Assert(dumpId == te->dumpId); - Assert(nBytes == strlen(str)); - - AH->public.n_errors += n_errors; - } - - return status; + return parallel_restore(AH, te); } diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 8dfc6a98de..9cadd0c4a4 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -152,9 +152,6 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) AH->ClonePtr = NULL; AH->DeClonePtr = NULL; - AH->MasterStartParallelItemPtr = NULL; - AH->MasterEndParallelItemPtr = NULL; - AH->WorkerJobDumpPtr = NULL; AH->WorkerJobRestorePtr = NULL; From 0109ab27609c0d58c1eddc6b799077d0968083de Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 27 Sep 2016 14:29:12 -0400 Subject: [PATCH 226/871] Make struct ParallelSlot private within pg_dump/parallel.c. The only field of this struct that other files have any need to touch is the pointer to the TocEntry a worker is working on. (Well, pg_backup_archiver.c is actually looking at workerStatus too, but that can be finessed by specifying that the TocEntry pointer is NULL for a non-busy worker.) Hence, move out the TocEntry pointers to a separate array within struct ParallelState, and then we can make struct ParallelSlot private. I noted the possibility of this previously, but hadn't got round to actually doing it. Discussion: <1188.1464544443@sss.pgh.pa.us> --- src/bin/pg_dump/parallel.c | 62 ++++++++++++++++++++++++---- src/bin/pg_dump/parallel.h | 45 +++----------------- src/bin/pg_dump/pg_backup_archiver.c | 12 +++--- 3 files changed, 65 insertions(+), 54 deletions(-) diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index 0e2bfa106a..5630dc626d 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -45,6 +45,8 @@ * WRKR_IDLE: it's waiting for a command * WRKR_WORKING: it's working on a command * WRKR_TERMINATED: process ended + * The pstate->te[] entry for each worker is valid when it's in WRKR_WORKING + * state, and must be NULL in other states. */ #include "postgres_fe.h" @@ -71,6 +73,45 @@ #define NO_SLOT (-1) /* Failure result for GetIdleWorker() */ +/* Worker process statuses */ +typedef enum +{ + WRKR_IDLE, + WRKR_WORKING, + WRKR_TERMINATED +} T_WorkerStatus; + +/* + * Private per-parallel-worker state (typedef for this is in parallel.h). + * + * Much of this is valid only in the master process (or, on Windows, should + * be touched only by the master thread). But the AH field should be touched + * only by workers. The pipe descriptors are valid everywhere. + */ +struct ParallelSlot +{ + T_WorkerStatus workerStatus; /* see enum above */ + + /* These fields are valid if workerStatus == WRKR_WORKING: */ + ParallelCompletionPtr callback; /* function to call on completion */ + void *callback_data; /* passthru data for it */ + + ArchiveHandle *AH; /* Archive data worker is using */ + + int pipeRead; /* master's end of the pipes */ + int pipeWrite; + int pipeRevRead; /* child's end of the pipes */ + int pipeRevWrite; + + /* Child process/thread identity info: */ +#ifdef WIN32 + uintptr_t hThread; + unsigned int threadId; +#else + pid_t pid; +#endif +}; + #ifdef WIN32 /* @@ -475,9 +516,10 @@ WaitForTerminatingWorkers(ParallelState *pstate) } #endif /* WIN32 */ - /* On all platforms, update workerStatus as well */ + /* On all platforms, update workerStatus and te[] as well */ Assert(j < pstate->numWorkers); slot->workerStatus = WRKR_TERMINATED; + pstate->te[j] = NULL; } } @@ -870,20 +912,22 @@ ParallelBackupStart(ArchiveHandle *AH) { ParallelState *pstate; int i; - const size_t slotSize = AH->public.numWorkers * sizeof(ParallelSlot); Assert(AH->public.numWorkers > 0); pstate = (ParallelState *) pg_malloc(sizeof(ParallelState)); pstate->numWorkers = AH->public.numWorkers; + pstate->te = NULL; pstate->parallelSlot = NULL; if (AH->public.numWorkers == 1) return pstate; - pstate->parallelSlot = (ParallelSlot *) pg_malloc(slotSize); - memset((void *) pstate->parallelSlot, 0, slotSize); + pstate->te = (TocEntry **) + pg_malloc0(pstate->numWorkers * sizeof(TocEntry *)); + pstate->parallelSlot = (ParallelSlot *) + pg_malloc0(pstate->numWorkers * sizeof(ParallelSlot)); #ifdef WIN32 /* Make fmtId() and fmtQualifiedId() use thread-local storage */ @@ -929,9 +973,10 @@ ParallelBackupStart(ArchiveHandle *AH) "could not create communication channels: %s\n", strerror(errno)); + pstate->te[i] = NULL; /* just for safety */ + slot->workerStatus = WRKR_IDLE; slot->AH = NULL; - slot->te = NULL; slot->callback = NULL; slot->callback_data = NULL; @@ -1062,6 +1107,7 @@ ParallelBackupEnd(ArchiveHandle *AH, ParallelState *pstate) set_cancel_pstate(NULL); /* Release state (mere neatnik-ism, since we're about to terminate) */ + free(pstate->te); free(pstate->parallelSlot); free(pstate); } @@ -1201,9 +1247,9 @@ DispatchJobForTocEntry(ArchiveHandle *AH, /* Remember worker is busy, and which TocEntry it's working on */ pstate->parallelSlot[worker].workerStatus = WRKR_WORKING; - pstate->parallelSlot[worker].te = te; pstate->parallelSlot[worker].callback = callback; pstate->parallelSlot[worker].callback_data = callback_data; + pstate->te[worker] = te; } /* @@ -1394,13 +1440,13 @@ ListenToWorkers(ArchiveHandle *AH, ParallelState *pstate, bool do_wait) if (messageStartsWith(msg, "OK ")) { ParallelSlot *slot = &pstate->parallelSlot[worker]; - TocEntry *te = slot->te; + TocEntry *te = pstate->te[worker]; int status; status = parseWorkerResponse(AH, te, msg); slot->callback(AH, te, status, slot->callback_data); slot->workerStatus = WRKR_IDLE; - slot->te = NULL; + pstate->te[worker] = NULL; } else exit_horribly(modulename, diff --git a/src/bin/pg_dump/parallel.h b/src/bin/pg_dump/parallel.h index 8ee629b106..e0c442cf37 100644 --- a/src/bin/pg_dump/parallel.h +++ b/src/bin/pg_dump/parallel.h @@ -33,51 +33,16 @@ typedef enum WFW_ALL_IDLE } WFW_WaitOption; -/* Worker process statuses */ -typedef enum -{ - WRKR_IDLE, - WRKR_WORKING, - WRKR_TERMINATED -} T_WorkerStatus; - -/* - * Per-parallel-worker state of parallel.c. - * - * Much of this is valid only in the master process (or, on Windows, should - * be touched only by the master thread). But the AH field should be touched - * only by workers. The pipe descriptors are valid everywhere. - */ -typedef struct ParallelSlot -{ - T_WorkerStatus workerStatus; /* see enum above */ - - /* These fields are valid if workerStatus == WRKR_WORKING: */ - TocEntry *te; /* item being worked on */ - ParallelCompletionPtr callback; /* function to call on completion */ - void *callback_data; /* passthru data for it */ - - ArchiveHandle *AH; /* Archive data worker is using */ - - int pipeRead; /* master's end of the pipes */ - int pipeWrite; - int pipeRevRead; /* child's end of the pipes */ - int pipeRevWrite; - - /* Child process/thread identity info: */ -#ifdef WIN32 - uintptr_t hThread; - unsigned int threadId; -#else - pid_t pid; -#endif -} ParallelSlot; +/* ParallelSlot is an opaque struct known only within parallel.c */ +typedef struct ParallelSlot ParallelSlot; /* Overall state for parallel.c */ typedef struct ParallelState { int numWorkers; /* allowed number of workers */ - ParallelSlot *parallelSlot; /* array of numWorkers slots */ + /* these arrays have numWorkers entries, one per worker: */ + TocEntry **te; /* item being worked on, or NULL */ + ParallelSlot *parallelSlot; /* private info about each worker */ } ParallelState; #ifdef WIN32 diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index e19c24aec9..bba8b6ca9f 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -4027,8 +4027,10 @@ get_next_work_item(ArchiveHandle *AH, TocEntry *ready_list, for (k = 0; k < pstate->numWorkers; k++) { - if (pstate->parallelSlot[k].workerStatus == WRKR_WORKING && - pstate->parallelSlot[k].te->section == SECTION_DATA) + TocEntry *running_te = pstate->te[k]; + + if (running_te != NULL && + running_te->section == SECTION_DATA) count++; } if (pstate->numWorkers == 0 || count * 4 < pstate->numWorkers) @@ -4049,12 +4051,10 @@ get_next_work_item(ArchiveHandle *AH, TocEntry *ready_list, */ for (i = 0; i < pstate->numWorkers; i++) { - TocEntry *running_te; + TocEntry *running_te = pstate->te[i]; - if (pstate->parallelSlot[i].workerStatus != WRKR_WORKING) + if (running_te == NULL) continue; - running_te = pstate->parallelSlot[i].te; - if (has_lock_conflicts(te, running_te) || has_lock_conflicts(running_te, te)) { From 72daabc7a3e75788df862104b8f723513c2471ae Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 27 Sep 2016 18:43:36 -0400 Subject: [PATCH 227/871] Disallow pushing volatile quals past set-returning functions. Pushing an upper-level restriction clause into an unflattened subquery-in-FROM is okay when the subquery contains no SRFs in its targetlist, or when it does but the SRFs are unreferenced by the clause *and the clause is not volatile*. Otherwise, we're changing the number of times the clause is evaluated, which is bad for volatile quals, and possibly changing the result, since a volatile qual might succeed for some SRF output rows and not others despite not referencing any of the changing columns. (Indeed, if the clause is something like "random() > 0.5", the user is probably expecting exactly that behavior.) We had most of these restrictions down, but not the one about the upper clause not being volatile. Fix that, and add a regression test to illustrate the expected behavior. Although this is definitely a bug, it doesn't seem like back-patch material, since possibly some users don't realize that the broken behavior is broken and are relying on what happens now. Also, while the added test is quite cheap in the wake of commit a4c35ea1c, it would be much more expensive (or else messier) in older branches. Per report from Tom van Tilburg. Discussion: --- src/backend/optimizer/path/allpaths.c | 12 ++- src/test/regress/expected/subselect.out | 100 ++++++++++++++++++++++++ src/test/regress/sql/subselect.sql | 44 +++++++++++ 3 files changed, 154 insertions(+), 2 deletions(-) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 99b6bc8f5a..e42ef98d8d 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -2254,6 +2254,12 @@ standard_join_search(PlannerInfo *root, int levels_needed, List *initial_rels) * thereby changing the partition contents and thus the window functions' * results for rows that remain. * + * 5. If the subquery contains any set-returning functions in its targetlist, + * we cannot push volatile quals into it. That would push them below the SRFs + * and thereby change the number of times they are evaluated. Also, a + * volatile qual could succeed for some SRF output rows and fail for others, + * a behavior that cannot occur if it's evaluated before SRF expansion. + * * In addition, we make several checks on the subquery's output columns to see * if it is safe to reference them in pushed-down quals. If output column k * is found to be unsafe to reference, we set safetyInfo->unsafeColumns[k] @@ -2298,8 +2304,10 @@ subquery_is_pushdown_safe(Query *subquery, Query *topquery, if (subquery->limitOffset != NULL || subquery->limitCount != NULL) return false; - /* Check points 3 and 4 */ - if (subquery->distinctClause || subquery->hasWindowFuncs) + /* Check points 3, 4, and 5 */ + if (subquery->distinctClause || + subquery->hasWindowFuncs || + subquery->hasTargetSRFs) safetyInfo->unsafeVolatile = true; /* diff --git a/src/test/regress/expected/subselect.out b/src/test/regress/expected/subselect.out index 0fc93d9d72..eda319d24b 100644 --- a/src/test/regress/expected/subselect.out +++ b/src/test/regress/expected/subselect.out @@ -880,3 +880,103 @@ select nextval('ts1'); 11 (1 row) +-- +-- Check that volatile quals aren't pushed down past a set-returning function; +-- while a nonvolatile qual can be, if it doesn't reference the SRF. +-- +create function tattle(x int, y int) returns bool +volatile language plpgsql as $$ +begin + raise notice 'x = %, y = %', x, y; + return x > y; +end$$; +explain (verbose, costs off) +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, 8); + QUERY PLAN +---------------------------------------------------------- + Subquery Scan on ss + Output: x, u + Filter: tattle(ss.x, 8) + -> Result + Output: 9, unnest('{1,2,3,11,12,13}'::integer[]) +(5 rows) + +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, 8); +NOTICE: x = 9, y = 8 +NOTICE: x = 9, y = 8 +NOTICE: x = 9, y = 8 +NOTICE: x = 9, y = 8 +NOTICE: x = 9, y = 8 +NOTICE: x = 9, y = 8 + x | u +---+---- + 9 | 1 + 9 | 2 + 9 | 3 + 9 | 11 + 9 | 12 + 9 | 13 +(6 rows) + +-- if we pretend it's stable, we get different results: +alter function tattle(x int, y int) stable; +explain (verbose, costs off) +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, 8); + QUERY PLAN +---------------------------------------------------- + Result + Output: 9, unnest('{1,2,3,11,12,13}'::integer[]) + One-Time Filter: tattle(9, 8) +(3 rows) + +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, 8); +NOTICE: x = 9, y = 8 + x | u +---+---- + 9 | 1 + 9 | 2 + 9 | 3 + 9 | 11 + 9 | 12 + 9 | 13 +(6 rows) + +-- although even a stable qual should not be pushed down if it references SRF +explain (verbose, costs off) +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, u); + QUERY PLAN +---------------------------------------------------------- + Subquery Scan on ss + Output: x, u + Filter: tattle(ss.x, ss.u) + -> Result + Output: 9, unnest('{1,2,3,11,12,13}'::integer[]) +(5 rows) + +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, u); +NOTICE: x = 9, y = 1 +NOTICE: x = 9, y = 2 +NOTICE: x = 9, y = 3 +NOTICE: x = 9, y = 11 +NOTICE: x = 9, y = 12 +NOTICE: x = 9, y = 13 + x | u +---+--- + 9 | 1 + 9 | 2 + 9 | 3 +(3 rows) + +drop function tattle(x int, y int); diff --git a/src/test/regress/sql/subselect.sql b/src/test/regress/sql/subselect.sql index 2991223089..08eb825c54 100644 --- a/src/test/regress/sql/subselect.sql +++ b/src/test/regress/sql/subselect.sql @@ -481,3 +481,47 @@ select * from order by 1; select nextval('ts1'); + +-- +-- Check that volatile quals aren't pushed down past a set-returning function; +-- while a nonvolatile qual can be, if it doesn't reference the SRF. +-- +create function tattle(x int, y int) returns bool +volatile language plpgsql as $$ +begin + raise notice 'x = %, y = %', x, y; + return x > y; +end$$; + +explain (verbose, costs off) +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, 8); + +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, 8); + +-- if we pretend it's stable, we get different results: +alter function tattle(x int, y int) stable; + +explain (verbose, costs off) +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, 8); + +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, 8); + +-- although even a stable qual should not be pushed down if it references SRF +explain (verbose, costs off) +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, u); + +select * from + (select 9 as x, unnest(array[1,2,3,11,12,13]) as u) ss + where tattle(x, u); + +drop function tattle(x int, y int); From babe05bc2b781eb3eb84a18d7010d08277e2e399 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 28 Sep 2016 12:22:44 +0300 Subject: [PATCH 228/871] Turn password_encryption GUC into an enum. This makes the parameter easier to extend, to support other password-based authentication protocols than MD5. (SCRAM is being worked on.) The GUC still accepts on/off as aliases for "md5" and "plain", although we may want to remove those once we actually add support for another password hash type. Michael Paquier, reviewed by David Steele, with some further edits by me. Discussion: --- doc/src/sgml/config.sgml | 17 +++---- src/backend/commands/user.c | 18 ++++---- src/backend/utils/misc/guc.c | 44 +++++++++++++------ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/commands/user.h | 15 +++++-- 5 files changed, 62 insertions(+), 34 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index a848a7edd1..e826c19698 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1163,21 +1163,22 @@ include_dir 'conf.d' - password_encryption (boolean) + password_encryption (enum) password_encryption configuration parameter - When a password is specified in or - - without writing either ENCRYPTED or - UNENCRYPTED, this parameter determines whether the - password is to be encrypted. The default is on - (encrypt the password). + When a password is specified in or + without writing either ENCRYPTED + or UNENCRYPTED, this parameter determines whether the + password is to be encrypted. The default value is md5, which + stores the password as an MD5 hash. Setting this to plain stores + it in plaintext. on and off are also accepted, as + aliases for md5 and plain, respectively. + diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index 4027c89b14..adc6b99b21 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -44,7 +44,7 @@ Oid binary_upgrade_next_pg_authid_oid = InvalidOid; /* GUC parameter */ -extern bool Password_encryption; +int Password_encryption = PASSWORD_TYPE_MD5; /* Hook to check passwords in CreateRole() and AlterRole() */ check_password_hook_type check_password_hook = NULL; @@ -80,7 +80,7 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) ListCell *item; ListCell *option; char *password = NULL; /* user password */ - bool encrypt_password = Password_encryption; /* encrypt password? */ + int password_type = Password_encryption; char encrypted_password[MD5_PASSWD_LEN + 1]; bool issuper = false; /* Make the user a superuser? */ bool inherit = true; /* Auto inherit privileges? */ @@ -140,9 +140,9 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) parser_errposition(pstate, defel->location))); dpassword = defel; if (strcmp(defel->defname, "encryptedPassword") == 0) - encrypt_password = true; + password_type = PASSWORD_TYPE_MD5; else if (strcmp(defel->defname, "unencryptedPassword") == 0) - encrypt_password = false; + password_type = PASSWORD_TYPE_PLAINTEXT; } else if (strcmp(defel->defname, "sysid") == 0) { @@ -393,7 +393,7 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) if (password) { - if (!encrypt_password || isMD5(password)) + if (password_type == PASSWORD_TYPE_PLAINTEXT || isMD5(password)) new_record[Anum_pg_authid_rolpassword - 1] = CStringGetTextDatum(password); else @@ -505,7 +505,7 @@ AlterRole(AlterRoleStmt *stmt) ListCell *option; char *rolename = NULL; char *password = NULL; /* user password */ - bool encrypt_password = Password_encryption; /* encrypt password? */ + int password_type = Password_encryption; char encrypted_password[MD5_PASSWD_LEN + 1]; int issuper = -1; /* Make the user a superuser? */ int inherit = -1; /* Auto inherit privileges? */ @@ -550,9 +550,9 @@ AlterRole(AlterRoleStmt *stmt) errmsg("conflicting or redundant options"))); dpassword = defel; if (strcmp(defel->defname, "encryptedPassword") == 0) - encrypt_password = true; + password_type = PASSWORD_TYPE_MD5; else if (strcmp(defel->defname, "unencryptedPassword") == 0) - encrypt_password = false; + password_type = PASSWORD_TYPE_PLAINTEXT; } else if (strcmp(defel->defname, "superuser") == 0) { @@ -804,7 +804,7 @@ AlterRole(AlterRoleStmt *stmt) /* password */ if (password) { - if (!encrypt_password || isMD5(password)) + if (password_type == PASSWORD_TYPE_PLAINTEXT || isMD5(password)) new_record[Anum_pg_authid_rolpassword - 1] = CStringGetTextDatum(password); else diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index ce4eef950b..cced814d6a 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -34,6 +34,7 @@ #include "catalog/namespace.h" #include "commands/async.h" #include "commands/prepare.h" +#include "commands/user.h" #include "commands/vacuum.h" #include "commands/variable.h" #include "commands/trigger.h" @@ -393,6 +394,24 @@ static const struct config_enum_entry force_parallel_mode_options[] = { {NULL, 0, false} }; +/* + * password_encryption used to be a boolean, so accept all the likely + * variants of "on" and "off", too. + */ +static const struct config_enum_entry password_encryption_options[] = { + {"plain", PASSWORD_TYPE_PLAINTEXT, false}, + {"md5", PASSWORD_TYPE_MD5, false}, + {"off", PASSWORD_TYPE_PLAINTEXT, false}, + {"on", PASSWORD_TYPE_MD5, false}, + {"true", PASSWORD_TYPE_MD5, true}, + {"false", PASSWORD_TYPE_PLAINTEXT, true}, + {"yes", PASSWORD_TYPE_MD5, true}, + {"no", PASSWORD_TYPE_PLAINTEXT, true}, + {"1", PASSWORD_TYPE_MD5, true}, + {"0", PASSWORD_TYPE_PLAINTEXT, true}, + {NULL, 0, false} +}; + /* * Options for enum values stored in other modules */ @@ -423,8 +442,6 @@ bool check_function_bodies = true; bool default_with_oids = false; bool SQL_inheritance = true; -bool Password_encryption = true; - int log_min_error_statement = ERROR; int log_min_messages = WARNING; int client_min_messages = NOTICE; @@ -1313,17 +1330,6 @@ static struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, - { - {"password_encryption", PGC_USERSET, CONN_AUTH_SECURITY, - gettext_noop("Encrypt passwords."), - gettext_noop("When a password is specified in CREATE USER or " - "ALTER USER without writing either ENCRYPTED or UNENCRYPTED, " - "this parameter determines whether the password is to be encrypted.") - }, - &Password_encryption, - true, - NULL, NULL, NULL - }, { {"transform_null_equals", PGC_USERSET, COMPAT_OPTIONS_CLIENT, gettext_noop("Treats \"expr=NULL\" as \"expr IS NULL\"."), @@ -3810,6 +3816,18 @@ static struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"password_encryption", PGC_USERSET, CONN_AUTH_SECURITY, + gettext_noop("Encrypt passwords."), + gettext_noop("When a password is specified in CREATE USER or " + "ALTER USER without writing either ENCRYPTED or UNENCRYPTED, " + "this parameter determines whether the password is to be encrypted.") + }, + &Password_encryption, + PASSWORD_TYPE_MD5, password_encryption_options, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, NULL, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b1c3aea9ee..05b1373594 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -85,7 +85,7 @@ #ssl_key_file = 'server.key' # (change requires restart) #ssl_ca_file = '' # (change requires restart) #ssl_crl_file = '' # (change requires restart) -#password_encryption = on +#password_encryption = md5 # md5 or plain #db_user_namespace = off #row_security = on diff --git a/src/include/commands/user.h b/src/include/commands/user.h index 1f0cfcc86f..102c2a5861 100644 --- a/src/include/commands/user.h +++ b/src/include/commands/user.h @@ -16,10 +16,19 @@ #include "parser/parse_node.h" -/* Hook to check passwords in CreateRole() and AlterRole() */ -#define PASSWORD_TYPE_PLAINTEXT 0 -#define PASSWORD_TYPE_MD5 1 +/* + * Types of password, for Password_encryption GUC and the password_type + * argument of the check-password hook. + */ +typedef enum PasswordType +{ + PASSWORD_TYPE_PLAINTEXT = 0, + PASSWORD_TYPE_MD5 +} PasswordType; +extern int Password_encryption; /* GUC */ + +/* Hook to check passwords in CreateRole() and AlterRole() */ typedef void (*check_password_hook_type) (const char *username, const char *password, int password_type, Datum validuntil_time, bool validuntil_null); extern PGDLLIMPORT check_password_hook_type check_password_hook; From 308985b0b404a5891a1a629f38cc46c2b2dcb4be Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 28 Sep 2016 11:19:46 -0400 Subject: [PATCH 229/871] Fix dangling pointer problem in ReorderBufferSerializeChange. Commit 3fe3511d05127cc024b221040db2eeb352e7d716 introduced a new case into this function, but neglected to ensure that the "ondisk" pointer got updated after a possible reallocation as the code does in other cases. Stas Kelvich, per diagnosis by Konstantin Knizhnik. --- src/backend/replication/logical/reorderbuffer.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 9b430b9acb..e2a502c443 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -2250,6 +2250,9 @@ ReorderBufferSerializeChange(ReorderBuffer *rb, ReorderBufferTXN *txn, data = ((char *) rb->outbuf) + sizeof(ReorderBufferDiskChange); + /* might have been reallocated above */ + ondisk = (ReorderBufferDiskChange *) rb->outbuf; + /* write the prefix including the size */ memcpy(data, &prefix_size, sizeof(Size)); data += sizeof(Size); From e79e6c4da152154544913b38354b98d07b65e411 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 28 Sep 2016 12:00:00 -0400 Subject: [PATCH 230/871] Fix CRC check handling in get_controlfile The previous patch broke this by returning NULL for a failed CRC check, which pg_controldata would then try to read. Fix by returning the result of the CRC check in a separate argument. Michael Paquier and myself --- src/backend/utils/misc/pg_controldata.c | 20 ++++++++++++-------- src/bin/pg_controldata/pg_controldata.c | 5 +++-- src/bin/pg_ctl/pg_ctl.c | 24 +++++++----------------- src/common/controldata_utils.c | 19 +++++++++---------- src/include/common/controldata_utils.h | 2 +- 5 files changed, 32 insertions(+), 38 deletions(-) diff --git a/src/backend/utils/misc/pg_controldata.c b/src/backend/utils/misc/pg_controldata.c index 4f9de83097..0c67833f60 100644 --- a/src/backend/utils/misc/pg_controldata.c +++ b/src/backend/utils/misc/pg_controldata.c @@ -34,6 +34,7 @@ pg_control_system(PG_FUNCTION_ARGS) TupleDesc tupdesc; HeapTuple htup; ControlFileData *ControlFile; + bool crc_ok; /* * Construct a tuple descriptor for the result row. This must match this @@ -51,8 +52,8 @@ pg_control_system(PG_FUNCTION_ARGS) tupdesc = BlessTupleDesc(tupdesc); /* read the control file */ - ControlFile = get_controlfile(DataDir, NULL); - if (!ControlFile) + ControlFile = get_controlfile(DataDir, NULL, &crc_ok); + if (!crc_ok) ereport(ERROR, (errmsg("calculated CRC checksum does not match value stored in file"))); @@ -83,6 +84,7 @@ pg_control_checkpoint(PG_FUNCTION_ARGS) ControlFileData *ControlFile; XLogSegNo segno; char xlogfilename[MAXFNAMELEN]; + bool crc_ok; /* * Construct a tuple descriptor for the result row. This must match this @@ -130,8 +132,8 @@ pg_control_checkpoint(PG_FUNCTION_ARGS) tupdesc = BlessTupleDesc(tupdesc); /* Read the control file. */ - ControlFile = get_controlfile(DataDir, NULL); - if (!ControlFile) + ControlFile = get_controlfile(DataDir, NULL, &crc_ok); + if (!crc_ok) ereport(ERROR, (errmsg("calculated CRC checksum does not match value stored in file"))); @@ -216,6 +218,7 @@ pg_control_recovery(PG_FUNCTION_ARGS) TupleDesc tupdesc; HeapTuple htup; ControlFileData *ControlFile; + bool crc_ok; /* * Construct a tuple descriptor for the result row. This must match this @@ -235,8 +238,8 @@ pg_control_recovery(PG_FUNCTION_ARGS) tupdesc = BlessTupleDesc(tupdesc); /* read the control file */ - ControlFile = get_controlfile(DataDir, NULL); - if (!ControlFile) + ControlFile = get_controlfile(DataDir, NULL, &crc_ok); + if (!crc_ok) ereport(ERROR, (errmsg("calculated CRC checksum does not match value stored in file"))); @@ -268,6 +271,7 @@ pg_control_init(PG_FUNCTION_ARGS) TupleDesc tupdesc; HeapTuple htup; ControlFileData *ControlFile; + bool crc_ok; /* * Construct a tuple descriptor for the result row. This must match this @@ -303,8 +307,8 @@ pg_control_init(PG_FUNCTION_ARGS) tupdesc = BlessTupleDesc(tupdesc); /* read the control file */ - ControlFile = get_controlfile(DataDir, NULL); - if (!ControlFile) + ControlFile = get_controlfile(DataDir, NULL, &crc_ok); + if (!crc_ok) ereport(ERROR, (errmsg("calculated CRC checksum does not match value stored in file"))); diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index e92feabade..20077a6639 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -86,6 +86,7 @@ int main(int argc, char *argv[]) { ControlFileData *ControlFile; + bool crc_ok; char *DataDir = NULL; time_t time_tmp; char pgctime_str[128]; @@ -155,8 +156,8 @@ main(int argc, char *argv[]) } /* get a copy of the control file */ - ControlFile = get_controlfile(DataDir, progname); - if (!ControlFile) + ControlFile = get_controlfile(DataDir, progname, &crc_ok); + if (!crc_ok) printf(_("WARNING: Calculated CRC checksum does not match value stored in file.\n" "Either the file is corrupt, or it has a different layout than this program\n" "is expecting. The results below are untrustworthy.\n\n")); diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index 052b02e0ff..ab10d2f25c 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -2147,28 +2147,18 @@ static DBState get_control_dbstate(void) { DBState ret; + bool crc_ok; + ControlFileData *control_file_data = get_controlfile(pg_data, progname, &crc_ok); - for (;;) + if (!crc_ok) { - ControlFileData *control_file_data = get_controlfile(pg_data, progname); - - if (control_file_data) - { - ret = control_file_data->state; - pfree(control_file_data); - return ret; - } - - if (wait_seconds > 0) - { - pg_usleep(1000000); /* 1 sec */ - wait_seconds--; - continue; - } - write_stderr(_("%s: control file appears to be corrupt\n"), progname); exit(1); } + + ret = control_file_data->state; + pfree(control_file_data); + return ret; } diff --git a/src/common/controldata_utils.c b/src/common/controldata_utils.c index f218d2558c..cbdae052a7 100644 --- a/src/common/controldata_utils.c +++ b/src/common/controldata_utils.c @@ -29,21 +29,24 @@ #include "port/pg_crc32c.h" /* - * get_controlfile(char *DataDir, const char *progname) + * get_controlfile(char *DataDir, const char *progname, bool *crc_ok_p) * - * Get controlfile values. The caller is responsible - * for pfreeing the result. + * Get controlfile values. The result is returned as a palloc'd copy of the + * control file data. * - * Returns NULL if the CRC did not match. + * crc_ok_p can be used by the caller to see whether the CRC of the control + * file data is correct. */ ControlFileData * -get_controlfile(const char *DataDir, const char *progname) +get_controlfile(const char *DataDir, const char *progname, bool *crc_ok_p) { ControlFileData *ControlFile; int fd; char ControlFilePath[MAXPGPATH]; pg_crc32c crc; + AssertArg(crc_ok_p); + ControlFile = palloc(sizeof(ControlFileData)); snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", DataDir); @@ -83,11 +86,7 @@ get_controlfile(const char *DataDir, const char *progname) offsetof(ControlFileData, crc)); FIN_CRC32C(crc); - if (!EQ_CRC32C(crc, ControlFile->crc)) - { - pfree(ControlFile); - return NULL; - } + *crc_ok_p = EQ_CRC32C(crc, ControlFile->crc); /* Make sure the control file is valid byte order. */ if (ControlFile->pg_control_version % 65536 == 0 && diff --git a/src/include/common/controldata_utils.h b/src/include/common/controldata_utils.h index f834624e4e..a822cba192 100644 --- a/src/include/common/controldata_utils.h +++ b/src/include/common/controldata_utils.h @@ -12,6 +12,6 @@ #include "catalog/pg_control.h" -extern ControlFileData *get_controlfile(const char *DataDir, const char *progname); +extern ControlFileData *get_controlfile(const char *DataDir, const char *progname, bool *crc_ok_p); #endif /* COMMON_CONTROLDATA_UTILS_H */ From 4929704acb3b4cdde8e890af977cdacaf1ac2ccb Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 28 Sep 2016 12:38:33 -0400 Subject: [PATCH 231/871] worker_spi: Call pgstat_report_stat. Without this, statistics changes accumulated by the worker never get reported to the stats collector, which is bad. Julien Rouhaud --- src/test/modules/worker_spi/worker_spi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/modules/worker_spi/worker_spi.c b/src/test/modules/worker_spi/worker_spi.c index 314e371b7f..7c9a3eb67e 100644 --- a/src/test/modules/worker_spi/worker_spi.c +++ b/src/test/modules/worker_spi/worker_spi.c @@ -292,6 +292,7 @@ worker_spi_main(Datum main_arg) SPI_finish(); PopActiveSnapshot(); CommitTransactionCommand(); + pgstat_report_stat(false); pgstat_report_activity(STATE_IDLE, NULL); } From 967ed9205bfcd89c1dde1e85735b32b404721399 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 28 Sep 2016 12:00:00 -0400 Subject: [PATCH 232/871] Remove dead line of code --- src/backend/commands/alter.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index 1301bcb5e8..03c0433992 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -757,7 +757,6 @@ ExecAlterOwnerStmt(AlterOwnerStmt *stmt) case OBJECT_TYPE: case OBJECT_DOMAIN: /* same as TYPE */ return AlterTypeOwner(stmt->object, newowner, stmt->objectType); - break; case OBJECT_FDW: return AlterForeignDataWrapperOwner(strVal(linitial(stmt->object)), From d3cd36a133d96ad5578b6c10279b55fd5b538093 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 28 Sep 2016 14:36:04 -0400 Subject: [PATCH 233/871] Make to_timestamp() and to_date() range-check fields of their input. Historically, something like to_date('2009-06-40','YYYY-MM-DD') would return '2009-07-10' because there was no prohibition on out-of-range month or day numbers. This has been widely panned, and it also turns out that Oracle throws an error in such cases. Since these functions are nominally Oracle-compatibility features, let's change that. There's no particular restriction on year (modulo the fact that the scanner may not believe that more than 4 digits are year digits, a matter to be addressed separately if at all). But we now check month, day, hour, minute, second, and fractional-second fields, as well as day-of-year and second-of-day fields if those are used. Currently, no checks are made on ISO-8601-style week numbers or day numbers; it's not very clear what the appropriate rules would be there, and they're probably so little used that it's not worth sweating over. Artur Zakirov, reviewed by Amul Sul, further adjustments by me Discussion: <1873520224.1784572.1465833145330.JavaMail.yahoo@mail.yahoo.com> See-Also: <57786490.9010201@wars-nicht.de> --- doc/src/sgml/func.sgml | 85 +++++++++++++---------- src/backend/utils/adt/formatting.c | 94 +++++++++++++++++-------- src/test/regress/expected/horology.out | 96 +++++++++++++++++++++++++- src/test/regress/sql/horology.sql | 30 +++++++- 4 files changed, 239 insertions(+), 66 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 67de029c6a..a58835082b 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -5832,6 +5832,17 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); + + + to_timestamp and to_date + exist to handle input formats that cannot be converted by + simple casting. For most standard date/time formats, simply casting the + source string to the required data type works, and is much easier. + Similarly, to_number is unnecessary for standard numeric + representations. + + + In a to_char output template string, there are certain patterns that are recognized and replaced with appropriately-formatted @@ -6038,7 +6049,7 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); Q - quarter (ignored by to_date and to_timestamp) + quarter RM @@ -6156,20 +6167,6 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - - - to_timestamp and to_date - exist to handle input formats that cannot be converted by - simple casting. These functions interpret input liberally, - with minimal error checking. While they produce valid output, - the conversion can yield unexpected results. For example, - input to these functions is not restricted by normal ranges, - thus to_date('20096040','YYYYMMDD') returns - 2014-01-17 rather than causing an error. - Casting does not have this behavior. - - - Ordinary text is allowed in to_char @@ -6195,7 +6192,8 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - If the year format specification is less than four digits, e.g. + In to_timestamp and to_date, + if the year format specification is less than four digits, e.g. YYY, and the supplied year is less than four digits, the year will be adjusted to be nearest to the year 2020, e.g. 95 becomes 1995. @@ -6204,8 +6202,9 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - The YYYY conversion from string to timestamp or - date has a restriction when processing years with more than 4 digits. You must + In to_timestamp and to_date, + the YYYY conversion has a restriction when + processing years with more than 4 digits. You must use some non-digit character or template after YYYY, otherwise the year is always interpreted as 4 digits. For example (with the year 20000): @@ -6219,12 +6218,12 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - In conversions from string to timestamp or - date, the CC (century) field is ignored + In to_timestamp and to_date, + the CC (century) field is accepted but ignored if there is a YYY, YYYY or Y,YYY field. If CC is used with - YY or Y then the year is computed - as the year in the specified century. If the century is + YY or Y then the result is + computed as that year in the specified century. If the century is specified but the year is not, the first year of the century is assumed. @@ -6232,9 +6231,19 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - An ISO 8601 week-numbering date (as distinct from a Gregorian date) - can be specified to to_timestamp and - to_date in one of two ways: + In to_timestamp and to_date, + weekday names or numbers (DAY, D, + and related field types) are accepted but are ignored for purposes of + computing the result. The same is true for quarter + (Q) fields. + + + + + + In to_timestamp and to_date, + an ISO 8601 week-numbering date (as distinct from a Gregorian date) + can be specified in one of two ways: @@ -6276,23 +6285,24 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - In a conversion from string to timestamp, millisecond + In to_timestamp, millisecond (MS) or microsecond (US) - values are used as the + fields are used as the seconds digits after the decimal point. For example - to_timestamp('12:3', 'SS:MS') is not 3 milliseconds, - but 300, because the conversion counts it as 12 + 0.3 seconds. - This means for the format SS:MS, the input values - 12:3, 12:30, and 12:300 specify the - same number of milliseconds. To get three milliseconds, one must use - 12:003, which the conversion counts as + to_timestamp('12.3', 'SS.MS') is not 3 milliseconds, + but 300, because the conversion treats it as 12 + 0.3 seconds. + So, for the format SS.MS, the input values + 12.3, 12.30, + and 12.300 specify the + same number of milliseconds. To get three milliseconds, one must write + 12.003, which the conversion treats as 12 + 0.003 = 12.003 seconds. Here is a more complex example: - to_timestamp('15:12:02.020.001230', 'HH:MI:SS.MS.US') + to_timestamp('15:12:02.020.001230', 'HH24:MI:SS.MS.US') is 15 hours, 12 minutes, and 2 seconds + 20 milliseconds + 1230 microseconds = 2.021230 seconds. @@ -6310,9 +6320,10 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); to_char(interval) formats HH and - HH12 as shown on a 12-hour clock, i.e. zero hours - and 36 hours output as 12, while HH24 - outputs the full hour value, which can exceed 23 for intervals. + HH12 as shown on a 12-hour clock, for example zero hours + and 36 hours both output as 12, while HH24 + outputs the full hour value, which can exceed 23 in + an interval value. diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index bbd97dc84b..d2d23d31ff 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -3553,9 +3553,6 @@ to_date(PG_FUNCTION_ARGS) * * The TmFromChar is then analysed and converted into the final results in * struct 'tm' and 'fsec'. - * - * This function does very little error checking, e.g. - * to_timestamp('20096040','YYYYMMDD') works */ static void do_to_timestamp(text *date_txt, text *fmt, @@ -3564,30 +3561,35 @@ do_to_timestamp(text *date_txt, text *fmt, FormatNode *format; TmFromChar tmfc; int fmt_len; + char *date_str; + int fmask; + + date_str = text_to_cstring(date_txt); ZERO_tmfc(&tmfc); ZERO_tm(tm); *fsec = 0; + fmask = 0; /* bit mask for ValidateDate() */ fmt_len = VARSIZE_ANY_EXHDR(fmt); if (fmt_len) { char *fmt_str; - char *date_str; bool incache; fmt_str = text_to_cstring(fmt); - /* - * Allocate new memory if format picture is bigger than static cache - * and not use cache (call parser always) - */ if (fmt_len > DCH_CACHE_SIZE) { - format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode)); + /* + * Allocate new memory if format picture is bigger than static + * cache and not use cache (call parser always) + */ incache = FALSE; + format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode)); + parse_format(format, fmt_str, DCH_keywords, DCH_suff, DCH_index, DCH_TYPE, NULL); @@ -3604,33 +3606,27 @@ do_to_timestamp(text *date_txt, text *fmt, if ((ent = DCH_cache_search(fmt_str)) == NULL) { - ent = DCH_cache_getnew(fmt_str); - /* * Not in the cache, must run parser and save a new * format-picture to the cache. */ + ent = DCH_cache_getnew(fmt_str); + parse_format(ent->format, fmt_str, DCH_keywords, DCH_suff, DCH_index, DCH_TYPE, NULL); (ent->format + fmt_len)->type = NODE_TYPE_END; /* Paranoia? */ -#ifdef DEBUG_TO_FROM_CHAR - /* dump_node(ent->format, fmt_len); */ - /* dump_index(DCH_keywords, DCH_index); */ -#endif } format = ent->format; } #ifdef DEBUG_TO_FROM_CHAR /* dump_node(format, fmt_len); */ + /* dump_index(DCH_keywords, DCH_index); */ #endif - date_str = text_to_cstring(date_txt); - DCH_from_char(format, date_str, &tmfc); - pfree(date_str); pfree(fmt_str); if (!incache) pfree(format); @@ -3639,8 +3635,7 @@ do_to_timestamp(text *date_txt, text *fmt, DEBUG_TMFC(&tmfc); /* - * Convert values that user define for FROM_CHAR (to_date/to_timestamp) to - * standard 'tm' + * Convert to_date/to_timestamp input fields to standard 'tm' */ if (tmfc.ssss) { @@ -3696,19 +3691,23 @@ do_to_timestamp(text *date_txt, text *fmt, tm->tm_year = (tmfc.cc + 1) * 100 - tm->tm_year + 1; } else + { /* find century year for dates ending in "00" */ tm->tm_year = tmfc.cc * 100 + ((tmfc.cc >= 0) ? 0 : 1); + } } else - /* If a 4-digit year is provided, we use that and ignore CC. */ { + /* If a 4-digit year is provided, we use that and ignore CC. */ tm->tm_year = tmfc.year; if (tmfc.bc && tm->tm_year > 0) tm->tm_year = -(tm->tm_year - 1); } + fmask |= DTK_M(YEAR); } - else if (tmfc.cc) /* use first year of century */ + else if (tmfc.cc) { + /* use first year of century */ if (tmfc.bc) tmfc.cc = -tmfc.cc; if (tmfc.cc >= 0) @@ -3717,10 +3716,14 @@ do_to_timestamp(text *date_txt, text *fmt, else /* +1 because year == 599 is 600 BC */ tm->tm_year = tmfc.cc * 100 + 1; + fmask |= DTK_M(YEAR); } if (tmfc.j) + { j2date(tmfc.j, &tm->tm_year, &tm->tm_mon, &tm->tm_mday); + fmask |= DTK_DATE_M; + } if (tmfc.ww) { @@ -3734,6 +3737,7 @@ do_to_timestamp(text *date_txt, text *fmt, isoweekdate2date(tmfc.ww, tmfc.d, &tm->tm_year, &tm->tm_mon, &tm->tm_mday); else isoweek2date(tmfc.ww, &tm->tm_year, &tm->tm_mon, &tm->tm_mday); + fmask |= DTK_DATE_M; } else tmfc.ddd = (tmfc.ww - 1) * 7 + 1; @@ -3741,14 +3745,16 @@ do_to_timestamp(text *date_txt, text *fmt, if (tmfc.w) tmfc.dd = (tmfc.w - 1) * 7 + 1; - if (tmfc.d) - tm->tm_wday = tmfc.d - 1; /* convert to native numbering */ if (tmfc.dd) + { tm->tm_mday = tmfc.dd; - if (tmfc.ddd) - tm->tm_yday = tmfc.ddd; + fmask |= DTK_M(DAY); + } if (tmfc.mm) + { tm->tm_mon = tmfc.mm; + fmask |= DTK_M(MONTH); + } if (tmfc.ddd && (tm->tm_mon <= 1 || tm->tm_mday <= 1)) { @@ -3771,6 +3777,7 @@ do_to_timestamp(text *date_txt, text *fmt, j0 = isoweek2j(tm->tm_year, 1) - 1; j2date(j0 + tmfc.ddd, &tm->tm_year, &tm->tm_mon, &tm->tm_mday); + fmask |= DTK_DATE_M; } else { @@ -3785,7 +3792,7 @@ do_to_timestamp(text *date_txt, text *fmt, for (i = 1; i <= MONTHS_PER_YEAR; i++) { - if (tmfc.ddd < y[i]) + if (tmfc.ddd <= y[i]) break; } if (tm->tm_mon <= 1) @@ -3793,6 +3800,8 @@ do_to_timestamp(text *date_txt, text *fmt, if (tm->tm_mday <= 1) tm->tm_mday = tmfc.ddd - y[i - 1]; + + fmask |= DTK_M(MONTH) | DTK_M(DAY); } } @@ -3808,7 +3817,38 @@ do_to_timestamp(text *date_txt, text *fmt, *fsec += (double) tmfc.us / 1000000; #endif + /* Range-check date fields according to bit mask computed above */ + if (fmask != 0) + { + /* We already dealt with AD/BC, so pass isjulian = true */ + int dterr = ValidateDate(fmask, true, false, false, tm); + + if (dterr != 0) + { + /* + * Force the error to be DTERR_FIELD_OVERFLOW even if ValidateDate + * said DTERR_MD_FIELD_OVERFLOW, because we don't want to print an + * irrelevant hint about datestyle. + */ + DateTimeParseError(DTERR_FIELD_OVERFLOW, date_str, "timestamp"); + } + } + + /* Range-check time fields too */ + if (tm->tm_hour < 0 || tm->tm_hour >= HOURS_PER_DAY || + tm->tm_min < 0 || tm->tm_min >= MINS_PER_HOUR || + tm->tm_sec < 0 || tm->tm_sec >= SECS_PER_MINUTE || +#ifdef HAVE_INT64_TIMESTAMP + *fsec < INT64CONST(0) || *fsec >= USECS_PER_SEC +#else + *fsec < 0 || *fsec >= 1 +#endif + ) + DateTimeParseError(DTERR_FIELD_OVERFLOW, date_str, "timestamp"); + DEBUG_TM(tm); + + pfree(date_str); } diff --git a/src/test/regress/expected/horology.out b/src/test/regress/expected/horology.out index 1fe02be093..f9d12e0f8a 100644 --- a/src/test/regress/expected/horology.out +++ b/src/test/regress/expected/horology.out @@ -2822,6 +2822,18 @@ SELECT to_timestamp('20000-1116', 'YYYY-MMDD'); Thu Nov 16 00:00:00 20000 PST (1 row) +SELECT to_timestamp('1997 AD 11 16', 'YYYY BC MM DD'); + to_timestamp +------------------------------ + Sun Nov 16 00:00:00 1997 PST +(1 row) + +SELECT to_timestamp('1997 BC 11 16', 'YYYY BC MM DD'); + to_timestamp +--------------------------------- + Tue Nov 16 00:00:00 1997 PST BC +(1 row) + SELECT to_timestamp('9-1116', 'Y-MMDD'); to_timestamp ------------------------------ @@ -2906,6 +2918,18 @@ SELECT to_timestamp(' 20050302', 'YYYYMMDD'); Wed Mar 02 00:00:00 2005 PST (1 row) +SELECT to_timestamp('2011-12-18 11:38 AM', 'YYYY-MM-DD HH12:MI PM'); + to_timestamp +------------------------------ + Sun Dec 18 11:38:00 2011 PST +(1 row) + +SELECT to_timestamp('2011-12-18 11:38 PM', 'YYYY-MM-DD HH12:MI PM'); + to_timestamp +------------------------------ + Sun Dec 18 23:38:00 2011 PST +(1 row) + -- -- Check handling of multiple spaces in format and/or input -- @@ -2982,7 +3006,7 @@ SELECT to_date('2011 12 18', 'YYYY MM DD'); (1 row) -- --- Check errors for some incorrect usages of to_timestamp() +-- Check errors for some incorrect usages of to_timestamp() and to_date() -- -- Mixture of date conventions (ISO week and Gregorian): SELECT to_timestamp('2005527', 'YYYYIWID'); @@ -3010,6 +3034,76 @@ DETAIL: Value must be an integer. SELECT to_timestamp('10000000000', 'FMYYYY'); ERROR: value for "YYYY" in source string is out of range DETAIL: Value must be in the range -2147483648 to 2147483647. +-- Out-of-range and not-quite-out-of-range fields: +SELECT to_timestamp('2016-06-13 25:00:00', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-06-13 25:00:00" +SELECT to_timestamp('2016-06-13 15:60:00', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-06-13 15:60:00" +SELECT to_timestamp('2016-06-13 15:50:60', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-06-13 15:50:60" +SELECT to_timestamp('2016-06-13 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); -- ok + to_timestamp +------------------------------ + Mon Jun 13 15:50:55 2016 PDT +(1 row) + +SELECT to_timestamp('2016-06-13 15:50:55', 'YYYY-MM-DD HH:MI:SS'); +ERROR: hour "15" is invalid for the 12-hour clock +HINT: Use the 24-hour clock, or give an hour between 1 and 12. +SELECT to_timestamp('2016-13-01 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-13-01 15:50:55" +SELECT to_timestamp('2016-02-30 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-02-30 15:50:55" +SELECT to_timestamp('2016-02-29 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); -- ok + to_timestamp +------------------------------ + Mon Feb 29 15:50:55 2016 PST +(1 row) + +SELECT to_timestamp('2015-02-29 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2015-02-29 15:50:55" +SELECT to_timestamp('2015-02-11 86000', 'YYYY-MM-DD SSSS'); -- ok + to_timestamp +------------------------------ + Wed Feb 11 23:53:20 2015 PST +(1 row) + +SELECT to_timestamp('2015-02-11 86400', 'YYYY-MM-DD SSSS'); +ERROR: date/time field value out of range: "2015-02-11 86400" +SELECT to_date('2016-13-10', 'YYYY-MM-DD'); +ERROR: date/time field value out of range: "2016-13-10" +SELECT to_date('2016-02-30', 'YYYY-MM-DD'); +ERROR: date/time field value out of range: "2016-02-30" +SELECT to_date('2016-02-29', 'YYYY-MM-DD'); -- ok + to_date +------------ + 02-29-2016 +(1 row) + +SELECT to_date('2015-02-29', 'YYYY-MM-DD'); +ERROR: date/time field value out of range: "2015-02-29" +SELECT to_date('2015 365', 'YYYY DDD'); -- ok + to_date +------------ + 12-31-2015 +(1 row) + +SELECT to_date('2015 366', 'YYYY DDD'); +ERROR: date/time field value out of range: "2015 366" +SELECT to_date('2016 365', 'YYYY DDD'); -- ok + to_date +------------ + 12-30-2016 +(1 row) + +SELECT to_date('2016 366', 'YYYY DDD'); -- ok + to_date +------------ + 12-31-2016 +(1 row) + +SELECT to_date('2016 367', 'YYYY DDD'); +ERROR: date/time field value out of range: "2016 367" -- -- Check behavior with SQL-style fixed-GMT-offset time zone (cf bug #8572) -- diff --git a/src/test/regress/sql/horology.sql b/src/test/regress/sql/horology.sql index c81437ba35..a7bc9dcfc4 100644 --- a/src/test/regress/sql/horology.sql +++ b/src/test/regress/sql/horology.sql @@ -412,6 +412,9 @@ SELECT to_timestamp('19971116', 'YYYYMMDD'); SELECT to_timestamp('20000-1116', 'YYYY-MMDD'); +SELECT to_timestamp('1997 AD 11 16', 'YYYY BC MM DD'); +SELECT to_timestamp('1997 BC 11 16', 'YYYY BC MM DD'); + SELECT to_timestamp('9-1116', 'Y-MMDD'); SELECT to_timestamp('95-1116', 'YY-MMDD'); @@ -440,6 +443,9 @@ SELECT to_timestamp(' 2005 03 02', 'YYYYMMDD'); SELECT to_timestamp(' 20050302', 'YYYYMMDD'); +SELECT to_timestamp('2011-12-18 11:38 AM', 'YYYY-MM-DD HH12:MI PM'); +SELECT to_timestamp('2011-12-18 11:38 PM', 'YYYY-MM-DD HH12:MI PM'); + -- -- Check handling of multiple spaces in format and/or input -- @@ -461,7 +467,7 @@ SELECT to_date('2011 12 18', 'YYYY MM DD'); SELECT to_date('2011 12 18', 'YYYY MM DD'); -- --- Check errors for some incorrect usages of to_timestamp() +-- Check errors for some incorrect usages of to_timestamp() and to_date() -- -- Mixture of date conventions (ISO week and Gregorian): @@ -482,6 +488,28 @@ SELECT to_timestamp('199711xy', 'YYYYMMDD'); -- Input that doesn't fit in an int: SELECT to_timestamp('10000000000', 'FMYYYY'); +-- Out-of-range and not-quite-out-of-range fields: +SELECT to_timestamp('2016-06-13 25:00:00', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-06-13 15:60:00', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-06-13 15:50:60', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-06-13 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); -- ok +SELECT to_timestamp('2016-06-13 15:50:55', 'YYYY-MM-DD HH:MI:SS'); +SELECT to_timestamp('2016-13-01 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-02-30 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-02-29 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); -- ok +SELECT to_timestamp('2015-02-29 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2015-02-11 86000', 'YYYY-MM-DD SSSS'); -- ok +SELECT to_timestamp('2015-02-11 86400', 'YYYY-MM-DD SSSS'); +SELECT to_date('2016-13-10', 'YYYY-MM-DD'); +SELECT to_date('2016-02-30', 'YYYY-MM-DD'); +SELECT to_date('2016-02-29', 'YYYY-MM-DD'); -- ok +SELECT to_date('2015-02-29', 'YYYY-MM-DD'); +SELECT to_date('2015 365', 'YYYY DDD'); -- ok +SELECT to_date('2015 366', 'YYYY DDD'); +SELECT to_date('2016 365', 'YYYY DDD'); -- ok +SELECT to_date('2016 366', 'YYYY DDD'); -- ok +SELECT to_date('2016 367', 'YYYY DDD'); + -- -- Check behavior with SQL-style fixed-GMT-offset time zone (cf bug #8572) -- From 83bed06be4e808f3da30f99b6c91e9efda3961ad Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 28 Sep 2016 17:08:40 -0400 Subject: [PATCH 234/871] Rationalize format-picture caching logic in formatting.c. Add a validity flag to DCHCacheEntry and NUMCacheEntry entries, and do not set it true until after we've parsed the supplied format string. This allows dealing with possible errors while parsing the format without the baroque hack that was there before (which only covered errors within NUMDesc_prepare, anyway). We can get rid of the PG_TRY in NUMDesc_prepare, as well as last_NUMCacheEntry and NUM_cache_remove. (Essentially, this reverts commit ff783fbae in favor of a less fragile solution; the problems with that approach are well illustrated by later hacking such as 55f927a46.) In passing, define the size of these caches as DCH_CACHE_ENTRIES not DCH_CACHE_FIELDS + 1 (whoever thought that was a good definition?) and likewise for the NUM cache. Also const-ify format string parameters where convenient, and merge duplicated cache lookup logic. This is primarily driven by a proposed patch from Artur Zakirov, which introduced some ereport's into format string parsing for the datetime case. He proposed preventing the creation of invalid cache entries by parsing the format string first into a local-variable array, and then copying that to a cache entry. That seemed a bit ugly to me, and anyway randomly different from the way the identical problem had been solved for the numeric case. Let's make the two sets of code more similar not less so. I'm not sure whether we'll adopt the new error conditions Artur proposes, but this patch seems like good code cleanup and future-proofing in any case. The existing code is critically (and undocumented-ly) dependent on no elog being thrown out of several nontrivial functions, which is trouble waiting to happen, though it doesn't seem to be actively broken today. Discussion: --- src/backend/utils/adt/formatting.c | 602 ++++++++++++++--------------- 1 file changed, 292 insertions(+), 310 deletions(-) diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index d2d23d31ff..d4eaa506e8 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -354,21 +354,27 @@ typedef struct /* ---------- * Format picture cache - * (cache size: - * Number part = NUM_CACHE_SIZE * NUM_CACHE_FIELDS - * Date-time part = DCH_CACHE_SIZE * DCH_CACHE_FIELDS - * ) + * + * We will cache datetime format pictures up to DCH_CACHE_SIZE bytes long; + * likewise number format pictures up to NUM_CACHE_SIZE bytes long. + * + * For simplicity, the cache entries are fixed-size, so they allow for the + * worst case of a FormatNode for each byte in the picture string. + * + * The max number of entries in the caches is DCH_CACHE_ENTRIES + * resp. NUM_CACHE_ENTRIES. * ---------- */ #define NUM_CACHE_SIZE 64 -#define NUM_CACHE_FIELDS 16 +#define NUM_CACHE_ENTRIES 20 #define DCH_CACHE_SIZE 128 -#define DCH_CACHE_FIELDS 16 +#define DCH_CACHE_ENTRIES 20 typedef struct { FormatNode format[DCH_CACHE_SIZE + 1]; char str[DCH_CACHE_SIZE + 1]; + bool valid; int age; } DCHCacheEntry; @@ -376,22 +382,20 @@ typedef struct { FormatNode format[NUM_CACHE_SIZE + 1]; char str[NUM_CACHE_SIZE + 1]; + bool valid; int age; NUMDesc Num; } NUMCacheEntry; -/* global cache for --- date/time part */ -static DCHCacheEntry DCHCache[DCH_CACHE_FIELDS + 1]; - -static int n_DCHCache = 0; /* number of entries */ -static int DCHCounter = 0; - -/* global cache for --- number part */ -static NUMCacheEntry NUMCache[NUM_CACHE_FIELDS + 1]; +/* global cache for date/time format pictures */ +static DCHCacheEntry DCHCache[DCH_CACHE_ENTRIES]; +static int n_DCHCache = 0; /* current number of entries */ +static int DCHCounter = 0; /* aging-event counter */ -static int n_NUMCache = 0; /* number of entries */ -static int NUMCounter = 0; -static NUMCacheEntry *last_NUMCacheEntry = NUMCache + 0; +/* global cache for number format pictures */ +static NUMCacheEntry NUMCache[NUM_CACHE_ENTRIES]; +static int n_NUMCache = 0; /* current number of entries */ +static int NUMCounter = 0; /* aging-event counter */ /* ---------- * For char->date/time conversion @@ -944,11 +948,11 @@ typedef struct NUMProc * Functions * ---------- */ -static const KeyWord *index_seq_search(char *str, const KeyWord *kw, +static const KeyWord *index_seq_search(const char *str, const KeyWord *kw, const int *index); -static const KeySuffix *suff_search(char *str, const KeySuffix *suf, int type); +static const KeySuffix *suff_search(const char *str, const KeySuffix *suf, int type); static void NUMDesc_prepare(NUMDesc *num, FormatNode *n); -static void parse_format(FormatNode *node, char *str, const KeyWord *kw, +static void parse_format(FormatNode *node, const char *str, const KeyWord *kw, const KeySuffix *suf, const int *index, int ver, NUMDesc *Num); static void DCH_to_char(FormatNode *node, bool is_interval, @@ -982,12 +986,12 @@ static void NUM_numpart_to_char(NUMProc *Np, int id); static char *NUM_processor(FormatNode *node, NUMDesc *Num, char *inout, char *number, int from_char_input_len, int to_char_out_pre_spaces, int sign, bool is_to_char, Oid collid); -static DCHCacheEntry *DCH_cache_search(char *str); -static DCHCacheEntry *DCH_cache_getnew(char *str); - -static NUMCacheEntry *NUM_cache_search(char *str); -static NUMCacheEntry *NUM_cache_getnew(char *str); -static void NUM_cache_remove(NUMCacheEntry *ent); +static DCHCacheEntry *DCH_cache_getnew(const char *str); +static DCHCacheEntry *DCH_cache_search(const char *str); +static DCHCacheEntry *DCH_cache_fetch(const char *str); +static NUMCacheEntry *NUM_cache_getnew(const char *str); +static NUMCacheEntry *NUM_cache_search(const char *str); +static NUMCacheEntry *NUM_cache_fetch(const char *str); /* ---------- @@ -997,7 +1001,7 @@ static void NUM_cache_remove(NUMCacheEntry *ent); * ---------- */ static const KeyWord * -index_seq_search(char *str, const KeyWord *kw, const int *index) +index_seq_search(const char *str, const KeyWord *kw, const int *index) { int poz; @@ -1021,7 +1025,7 @@ index_seq_search(char *str, const KeyWord *kw, const int *index) } static const KeySuffix * -suff_search(char *str, const KeySuffix *suf, int type) +suff_search(const char *str, const KeySuffix *suf, int type) { const KeySuffix *s; @@ -1046,182 +1050,166 @@ NUMDesc_prepare(NUMDesc *num, FormatNode *n) if (n->type != NODE_TYPE_ACTION) return; - /* - * In case of an error, we need to remove the numeric from the cache. Use - * a PG_TRY block to ensure that this happens. - */ - PG_TRY(); - { - if (IS_EEEE(num) && n->key->id != NUM_E) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("\"EEEE\" must be the last pattern used"))); - - switch (n->key->id) - { - case NUM_9: - if (IS_BRACKET(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("\"9\" must be ahead of \"PR\""))); - if (IS_MULTI(num)) - { - ++num->multi; - break; - } - if (IS_DECIMAL(num)) - ++num->post; - else - ++num->pre; - break; - - case NUM_0: - if (IS_BRACKET(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("\"0\" must be ahead of \"PR\""))); - if (!IS_ZERO(num) && !IS_DECIMAL(num)) - { - num->flag |= NUM_F_ZERO; - num->zero_start = num->pre + 1; - } - if (!IS_DECIMAL(num)) - ++num->pre; - else - ++num->post; - - num->zero_end = num->pre + num->post; - break; + if (IS_EEEE(num) && n->key->id != NUM_E) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("\"EEEE\" must be the last pattern used"))); - case NUM_B: - if (num->pre == 0 && num->post == 0 && (!IS_ZERO(num))) - num->flag |= NUM_F_BLANK; + switch (n->key->id) + { + case NUM_9: + if (IS_BRACKET(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("\"9\" must be ahead of \"PR\""))); + if (IS_MULTI(num)) + { + ++num->multi; break; + } + if (IS_DECIMAL(num)) + ++num->post; + else + ++num->pre; + break; - case NUM_D: - num->flag |= NUM_F_LDECIMAL; - num->need_locale = TRUE; - /* FALLTHROUGH */ - case NUM_DEC: - if (IS_DECIMAL(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("multiple decimal points"))); - if (IS_MULTI(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), + case NUM_0: + if (IS_BRACKET(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("\"0\" must be ahead of \"PR\""))); + if (!IS_ZERO(num) && !IS_DECIMAL(num)) + { + num->flag |= NUM_F_ZERO; + num->zero_start = num->pre + 1; + } + if (!IS_DECIMAL(num)) + ++num->pre; + else + ++num->post; + + num->zero_end = num->pre + num->post; + break; + + case NUM_B: + if (num->pre == 0 && num->post == 0 && (!IS_ZERO(num))) + num->flag |= NUM_F_BLANK; + break; + + case NUM_D: + num->flag |= NUM_F_LDECIMAL; + num->need_locale = TRUE; + /* FALLTHROUGH */ + case NUM_DEC: + if (IS_DECIMAL(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("multiple decimal points"))); + if (IS_MULTI(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot use \"V\" and decimal point together"))); - num->flag |= NUM_F_DECIMAL; - break; - - case NUM_FM: - num->flag |= NUM_F_FILLMODE; - break; + num->flag |= NUM_F_DECIMAL; + break; - case NUM_S: - if (IS_LSIGN(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("cannot use \"S\" twice"))); - if (IS_PLUS(num) || IS_MINUS(num) || IS_BRACKET(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"))); - if (!IS_DECIMAL(num)) - { - num->lsign = NUM_LSIGN_PRE; - num->pre_lsign_num = num->pre; - num->need_locale = TRUE; - num->flag |= NUM_F_LSIGN; - } - else if (num->lsign == NUM_LSIGN_NONE) - { - num->lsign = NUM_LSIGN_POST; - num->need_locale = TRUE; - num->flag |= NUM_F_LSIGN; - } - break; - - case NUM_MI: - if (IS_LSIGN(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("cannot use \"S\" and \"MI\" together"))); - num->flag |= NUM_F_MINUS; - if (IS_DECIMAL(num)) - num->flag |= NUM_F_MINUS_POST; - break; - - case NUM_PL: - if (IS_LSIGN(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("cannot use \"S\" and \"PL\" together"))); - num->flag |= NUM_F_PLUS; - if (IS_DECIMAL(num)) - num->flag |= NUM_F_PLUS_POST; - break; - - case NUM_SG: - if (IS_LSIGN(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("cannot use \"S\" and \"SG\" together"))); - num->flag |= NUM_F_MINUS; - num->flag |= NUM_F_PLUS; - break; - - case NUM_PR: - if (IS_LSIGN(num) || IS_PLUS(num) || IS_MINUS(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"))); - num->flag |= NUM_F_BRACKET; - break; - - case NUM_rn: - case NUM_RN: - num->flag |= NUM_F_ROMAN; - break; + case NUM_FM: + num->flag |= NUM_F_FILLMODE; + break; - case NUM_L: - case NUM_G: + case NUM_S: + if (IS_LSIGN(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot use \"S\" twice"))); + if (IS_PLUS(num) || IS_MINUS(num) || IS_BRACKET(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"))); + if (!IS_DECIMAL(num)) + { + num->lsign = NUM_LSIGN_PRE; + num->pre_lsign_num = num->pre; num->need_locale = TRUE; - break; + num->flag |= NUM_F_LSIGN; + } + else if (num->lsign == NUM_LSIGN_NONE) + { + num->lsign = NUM_LSIGN_POST; + num->need_locale = TRUE; + num->flag |= NUM_F_LSIGN; + } + break; - case NUM_V: - if (IS_DECIMAL(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), + case NUM_MI: + if (IS_LSIGN(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot use \"S\" and \"MI\" together"))); + num->flag |= NUM_F_MINUS; + if (IS_DECIMAL(num)) + num->flag |= NUM_F_MINUS_POST; + break; + + case NUM_PL: + if (IS_LSIGN(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot use \"S\" and \"PL\" together"))); + num->flag |= NUM_F_PLUS; + if (IS_DECIMAL(num)) + num->flag |= NUM_F_PLUS_POST; + break; + + case NUM_SG: + if (IS_LSIGN(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot use \"S\" and \"SG\" together"))); + num->flag |= NUM_F_MINUS; + num->flag |= NUM_F_PLUS; + break; + + case NUM_PR: + if (IS_LSIGN(num) || IS_PLUS(num) || IS_MINUS(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"))); + num->flag |= NUM_F_BRACKET; + break; + + case NUM_rn: + case NUM_RN: + num->flag |= NUM_F_ROMAN; + break; + + case NUM_L: + case NUM_G: + num->need_locale = TRUE; + break; + + case NUM_V: + if (IS_DECIMAL(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot use \"V\" and decimal point together"))); - num->flag |= NUM_F_MULTI; - break; + num->flag |= NUM_F_MULTI; + break; - case NUM_E: - if (IS_EEEE(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("cannot use \"EEEE\" twice"))); - if (IS_BLANK(num) || IS_FILLMODE(num) || IS_LSIGN(num) || - IS_BRACKET(num) || IS_MINUS(num) || IS_PLUS(num) || - IS_ROMAN(num) || IS_MULTI(num)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), + case NUM_E: + if (IS_EEEE(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot use \"EEEE\" twice"))); + if (IS_BLANK(num) || IS_FILLMODE(num) || IS_LSIGN(num) || + IS_BRACKET(num) || IS_MINUS(num) || IS_PLUS(num) || + IS_ROMAN(num) || IS_MULTI(num)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), errmsg("\"EEEE\" is incompatible with other formats"), - errdetail("\"EEEE\" may only be used together with digit and decimal point patterns."))); - num->flag |= NUM_F_EEEE; - break; - } - } - PG_CATCH(); - { - NUM_cache_remove(last_NUMCacheEntry); - PG_RE_THROW(); + errdetail("\"EEEE\" may only be used together with digit and decimal point patterns."))); + num->flag |= NUM_F_EEEE; + break; } - PG_END_TRY(); - - - return; } /* ---------- @@ -1232,7 +1220,7 @@ NUMDesc_prepare(NUMDesc *num, FormatNode *n) * ---------- */ static void -parse_format(FormatNode *node, char *str, const KeyWord *kw, +parse_format(FormatNode *node, const char *str, const KeyWord *kw, const KeySuffix *suf, const int *index, int ver, NUMDesc *Num) { const KeySuffix *s; @@ -1350,7 +1338,6 @@ parse_format(FormatNode *node, char *str, const KeyWord *kw, n->type = NODE_TYPE_END; n->suffix = 0; - return; } /* ---------- @@ -3210,41 +3197,51 @@ DCH_from_char(FormatNode *node, char *in, TmFromChar *out) } } +/* select a DCHCacheEntry to hold the given format picture */ static DCHCacheEntry * -DCH_cache_getnew(char *str) +DCH_cache_getnew(const char *str) { DCHCacheEntry *ent; /* counter overflow check - paranoia? */ - if (DCHCounter >= (INT_MAX - DCH_CACHE_FIELDS - 1)) + if (DCHCounter >= (INT_MAX - DCH_CACHE_ENTRIES)) { DCHCounter = 0; - for (ent = DCHCache; ent <= (DCHCache + DCH_CACHE_FIELDS); ent++) + for (ent = DCHCache; ent < (DCHCache + DCH_CACHE_ENTRIES); ent++) ent->age = (++DCHCounter); } /* - * If cache is full, remove oldest entry + * If cache is full, remove oldest entry (or recycle first not-valid one) */ - if (n_DCHCache > DCH_CACHE_FIELDS) + if (n_DCHCache >= DCH_CACHE_ENTRIES) { DCHCacheEntry *old = DCHCache + 0; #ifdef DEBUG_TO_FROM_CHAR elog(DEBUG_elog_output, "cache is full (%d)", n_DCHCache); #endif - for (ent = DCHCache + 1; ent <= (DCHCache + DCH_CACHE_FIELDS); ent++) + if (old->valid) { - if (ent->age < old->age) - old = ent; + for (ent = DCHCache + 1; ent < (DCHCache + DCH_CACHE_ENTRIES); ent++) + { + if (!ent->valid) + { + old = ent; + break; + } + if (ent->age < old->age) + old = ent; + } } #ifdef DEBUG_TO_FROM_CHAR elog(DEBUG_elog_output, "OLD: '%s' AGE: %d", old->str, old->age); #endif + old->valid = false; StrNCpy(old->str, str, DCH_CACHE_SIZE + 1); - /* old->format fill parser */ old->age = (++DCHCounter); + /* caller is expected to fill format, then set valid */ return old; } else @@ -3253,32 +3250,34 @@ DCH_cache_getnew(char *str) elog(DEBUG_elog_output, "NEW (%d)", n_DCHCache); #endif ent = DCHCache + n_DCHCache; + ent->valid = false; StrNCpy(ent->str, str, DCH_CACHE_SIZE + 1); - /* ent->format fill parser */ ent->age = (++DCHCounter); + /* caller is expected to fill format, then set valid */ ++n_DCHCache; return ent; } } +/* look for an existing DCHCacheEntry matching the given format picture */ static DCHCacheEntry * -DCH_cache_search(char *str) +DCH_cache_search(const char *str) { int i; DCHCacheEntry *ent; /* counter overflow check - paranoia? */ - if (DCHCounter >= (INT_MAX - DCH_CACHE_FIELDS - 1)) + if (DCHCounter >= (INT_MAX - DCH_CACHE_ENTRIES)) { DCHCounter = 0; - for (ent = DCHCache; ent <= (DCHCache + DCH_CACHE_FIELDS); ent++) + for (ent = DCHCache; ent < (DCHCache + DCH_CACHE_ENTRIES); ent++) ent->age = (++DCHCounter); } for (i = 0, ent = DCHCache; i < n_DCHCache; i++, ent++) { - if (strcmp(ent->str, str) == 0) + if (ent->valid && strcmp(ent->str, str) == 0) { ent->age = (++DCHCounter); return ent; @@ -3288,6 +3287,29 @@ DCH_cache_search(char *str) return NULL; } +/* Find or create a DCHCacheEntry for the given format picture */ +static DCHCacheEntry * +DCH_cache_fetch(const char *str) +{ + DCHCacheEntry *ent; + + if ((ent = DCH_cache_search(str)) == NULL) + { + /* + * Not in the cache, must run parser and save a new format-picture to + * the cache. Do not mark the cache entry valid until parsing + * succeeds. + */ + ent = DCH_cache_getnew(str); + + parse_format(ent->format, str, DCH_keywords, + DCH_suff, DCH_index, DCH_TYPE, NULL); + + ent->valid = true; + } + return ent; +} + /* * Format a date/time or interval into a string according to fmt. * We parse fmt into a list of FormatNodes. This is then passed to DCH_to_char @@ -3315,47 +3337,27 @@ datetime_to_char_body(TmToChar *tmtc, text *fmt, bool is_interval, Oid collid) result = palloc((fmt_len * DCH_MAX_ITEM_SIZ) + 1); *result = '\0'; - /* - * Allocate new memory if format picture is bigger than static cache and - * not use cache (call parser always) - */ if (fmt_len > DCH_CACHE_SIZE) { - format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode)); + /* + * Allocate new memory if format picture is bigger than static cache + * and do not use cache (call parser always) + */ incache = FALSE; + format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode)); + parse_format(format, fmt_str, DCH_keywords, DCH_suff, DCH_index, DCH_TYPE, NULL); - - (format + fmt_len)->type = NODE_TYPE_END; /* Paranoia? */ } else { /* * Use cache buffers */ - DCHCacheEntry *ent; + DCHCacheEntry *ent = DCH_cache_fetch(fmt_str); incache = TRUE; - - if ((ent = DCH_cache_search(fmt_str)) == NULL) - { - ent = DCH_cache_getnew(fmt_str); - - /* - * Not in the cache, must run parser and save a new format-picture - * to the cache. - */ - parse_format(ent->format, fmt_str, DCH_keywords, - DCH_suff, DCH_index, DCH_TYPE, NULL); - - (ent->format + fmt_len)->type = NODE_TYPE_END; /* Paranoia? */ - -#ifdef DEBUG_TO_FROM_CHAR - /* dump_node(ent->format, fmt_len); */ - /* dump_index(DCH_keywords, DCH_index); */ -#endif - } format = ent->format; } @@ -3584,7 +3586,7 @@ do_to_timestamp(text *date_txt, text *fmt, { /* * Allocate new memory if format picture is bigger than static - * cache and not use cache (call parser always) + * cache and do not use cache (call parser always) */ incache = FALSE; @@ -3592,31 +3594,15 @@ do_to_timestamp(text *date_txt, text *fmt, parse_format(format, fmt_str, DCH_keywords, DCH_suff, DCH_index, DCH_TYPE, NULL); - - (format + fmt_len)->type = NODE_TYPE_END; /* Paranoia? */ } else { /* * Use cache buffers */ - DCHCacheEntry *ent; + DCHCacheEntry *ent = DCH_cache_fetch(fmt_str); incache = TRUE; - - if ((ent = DCH_cache_search(fmt_str)) == NULL) - { - /* - * Not in the cache, must run parser and save a new - * format-picture to the cache. - */ - ent = DCH_cache_getnew(fmt_str); - - parse_format(ent->format, fmt_str, DCH_keywords, - DCH_suff, DCH_index, DCH_TYPE, NULL); - - (ent->format + fmt_len)->type = NODE_TYPE_END; /* Paranoia? */ - } format = ent->format; } @@ -3878,51 +3864,52 @@ do { \ (_n)->zero_end = 0; \ } while(0) +/* select a NUMCacheEntry to hold the given format picture */ static NUMCacheEntry * -NUM_cache_getnew(char *str) +NUM_cache_getnew(const char *str) { NUMCacheEntry *ent; /* counter overflow check - paranoia? */ - if (NUMCounter >= (INT_MAX - NUM_CACHE_FIELDS - 1)) + if (NUMCounter >= (INT_MAX - NUM_CACHE_ENTRIES)) { NUMCounter = 0; - for (ent = NUMCache; ent <= (NUMCache + NUM_CACHE_FIELDS); ent++) + for (ent = NUMCache; ent < (NUMCache + NUM_CACHE_ENTRIES); ent++) ent->age = (++NUMCounter); } /* - * If cache is full, remove oldest entry + * If cache is full, remove oldest entry (or recycle first not-valid one) */ - if (n_NUMCache > NUM_CACHE_FIELDS) + if (n_NUMCache >= NUM_CACHE_ENTRIES) { NUMCacheEntry *old = NUMCache + 0; #ifdef DEBUG_TO_FROM_CHAR elog(DEBUG_elog_output, "Cache is full (%d)", n_NUMCache); #endif - for (ent = NUMCache; ent <= (NUMCache + NUM_CACHE_FIELDS); ent++) + if (old->valid) { - /* - * entry removed via NUM_cache_remove() can be used here, which is - * why it's worth scanning first entry again - */ - if (ent->str[0] == '\0') + for (ent = NUMCache + 1; ent < (NUMCache + NUM_CACHE_ENTRIES); ent++) { - old = ent; - break; + if (!ent->valid) + { + old = ent; + break; + } + if (ent->age < old->age) + old = ent; } - if (ent->age < old->age) - old = ent; } #ifdef DEBUG_TO_FROM_CHAR elog(DEBUG_elog_output, "OLD: \"%s\" AGE: %d", old->str, old->age); #endif + old->valid = false; StrNCpy(old->str, str, NUM_CACHE_SIZE + 1); - /* old->format fill parser */ old->age = (++NUMCounter); - ent = old; + /* caller is expected to fill format and Num, then set valid */ + return old; } else { @@ -3930,39 +3917,36 @@ NUM_cache_getnew(char *str) elog(DEBUG_elog_output, "NEW (%d)", n_NUMCache); #endif ent = NUMCache + n_NUMCache; + ent->valid = false; StrNCpy(ent->str, str, NUM_CACHE_SIZE + 1); - /* ent->format fill parser */ ent->age = (++NUMCounter); + /* caller is expected to fill format and Num, then set valid */ ++n_NUMCache; + return ent; } - - zeroize_NUM(&ent->Num); - - last_NUMCacheEntry = ent; - return ent; } +/* look for an existing NUMCacheEntry matching the given format picture */ static NUMCacheEntry * -NUM_cache_search(char *str) +NUM_cache_search(const char *str) { int i; NUMCacheEntry *ent; /* counter overflow check - paranoia? */ - if (NUMCounter >= (INT_MAX - NUM_CACHE_FIELDS - 1)) + if (NUMCounter >= (INT_MAX - NUM_CACHE_ENTRIES)) { NUMCounter = 0; - for (ent = NUMCache; ent <= (NUMCache + NUM_CACHE_FIELDS); ent++) + for (ent = NUMCache; ent < (NUMCache + NUM_CACHE_ENTRIES); ent++) ent->age = (++NUMCounter); } for (i = 0, ent = NUMCache; i < n_NUMCache; i++, ent++) { - if (strcmp(ent->str, str) == 0) + if (ent->valid && strcmp(ent->str, str) == 0) { ent->age = (++NUMCounter); - last_NUMCacheEntry = ent; return ent; } } @@ -3970,14 +3954,29 @@ NUM_cache_search(char *str) return NULL; } -static void -NUM_cache_remove(NUMCacheEntry *ent) +/* Find or create a NUMCacheEntry for the given format picture */ +static NUMCacheEntry * +NUM_cache_fetch(const char *str) { -#ifdef DEBUG_TO_FROM_CHAR - elog(DEBUG_elog_output, "REMOVING ENTRY (%s)", ent->str); -#endif - ent->str[0] = '\0'; - ent->age = 0; + NUMCacheEntry *ent; + + if ((ent = NUM_cache_search(str)) == NULL) + { + /* + * Not in the cache, must run parser and save a new format-picture to + * the cache. Do not mark the cache entry valid until parsing + * succeeds. + */ + ent = NUM_cache_getnew(str); + + zeroize_NUM(&ent->Num); + + parse_format(ent->format, str, NUM_keywords, + NULL, NUM_index, NUM_TYPE, &ent->Num); + + ent->valid = true; + } + return ent; } /* ---------- @@ -3992,13 +3991,12 @@ NUM_cache(int len, NUMDesc *Num, text *pars_str, bool *shouldFree) str = text_to_cstring(pars_str); - /* - * Allocate new memory if format picture is bigger than static cache and - * not use cache (call parser always). This branches sets shouldFree to - * true, accordingly. - */ if (len > NUM_CACHE_SIZE) { + /* + * Allocate new memory if format picture is bigger than static cache + * and do not use cache (call parser always) + */ format = (FormatNode *) palloc((len + 1) * sizeof(FormatNode)); *shouldFree = true; @@ -4007,32 +4005,16 @@ NUM_cache(int len, NUMDesc *Num, text *pars_str, bool *shouldFree) parse_format(format, str, NUM_keywords, NULL, NUM_index, NUM_TYPE, Num); - - (format + len)->type = NODE_TYPE_END; /* Paranoia? */ } else { /* * Use cache buffers */ - NUMCacheEntry *ent; + NUMCacheEntry *ent = NUM_cache_fetch(str); *shouldFree = false; - if ((ent = NUM_cache_search(str)) == NULL) - { - ent = NUM_cache_getnew(str); - - /* - * Not in the cache, must run parser and save a new format-picture - * to the cache. - */ - parse_format(ent->format, str, NUM_keywords, - NULL, NUM_index, NUM_TYPE, &ent->Num); - - (ent->format + len)->type = NODE_TYPE_END; /* Paranoia? */ - } - format = ent->format; /* From b82d5a2c7c9dd1a01041feef2834db27a46fb2d3 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Wed, 28 Sep 2016 19:31:58 -0300 Subject: [PATCH 235/871] Silence compiler warnings Reported by Peter Eisentraut. Coding suggested by Tom Lane. --- src/backend/catalog/objectaddress.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 9aa81748ba..d531d17cdb 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -2290,23 +2290,18 @@ get_object_namespace(const ObjectAddress *address) int read_objtype_from_string(const char *objtype) { - ObjectType type; int i; for (i = 0; i < lengthof(ObjectTypeMap); i++) { if (strcmp(ObjectTypeMap[i].tm_name, objtype) == 0) - { - type = ObjectTypeMap[i].tm_type; - break; - } + return ObjectTypeMap[i].tm_type; } - if (i >= lengthof(ObjectTypeMap)) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("unrecognized object type \"%s\"", objtype))); + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unrecognized object type \"%s\"", objtype))); - return type; + return -1; /* keep compiler quiet */ } /* From 6ad8ac6026287e3ccbc4d606b6ab6116ccc0eec8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 28 Sep 2016 12:00:00 -0400 Subject: [PATCH 236/871] Exclude additional directories in pg_basebackup The list of files and directories that pg_basebackup excludes from the backup was somewhat incomplete and unorganized. Change that with having the exclusion driven from tables. Clean up some code around it. Also document the exclusions in more detail so that users of pg_start_backup can make use of it as well. The contents of these directories are now excluded from the backup: pg_dynshmem, pg_notify, pg_serial, pg_snapshots, pg_subtrans Also fix a bug that a pg_repl_slot or pg_stat_tmp being a symlink would cause a corrupt tar header to be created. Now such symlinks are included in the backup as empty directories. Bug found by Ashutosh Sharma . From: David Steele Reviewed-by: Michael Paquier --- doc/src/sgml/backup.sgml | 16 ++ doc/src/sgml/protocol.sgml | 13 +- doc/src/sgml/ref/pg_basebackup.sgml | 10 +- src/backend/replication/basebackup.c | 260 ++++++++++++------- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 44 +++- 5 files changed, 243 insertions(+), 100 deletions(-) diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 0f09d82d65..95d0ff3149 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -1089,6 +1089,22 @@ SELECT pg_stop_backup(); the new master comes on line. + + The contents of the directories pg_dynshmem/, + pg_notify/, pg_serial/, + pg_snapshots/, pg_stat_tmp/, + and pg_subtrans/ (but not the directories themselves) can be + omitted from the backup as they will be initialized on postmaster startup. + If is set and is under the data + directory then the contents of that directory can also be omitted. + + + + Any file or directory beginning with pgsql_tmp can be + omitted from the backup. These files are removed on postmaster start and + the directories will be recreated as needed. + + The backup label file includes the label string you gave to pg_start_backup, diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 68b0941029..3384e73448 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2069,7 +2069,9 @@ The commands accepted in walsender mode are: - various temporary files created during the operation of the PostgreSQL server + Various temporary files and directories created during the operation + of the PostgreSQL server, such as any file or directory beginning + with pgsql_tmp. @@ -2082,13 +2084,18 @@ The commands accepted in walsender mode are: - pg_replslot is copied as an empty directory. + pg_dynshmem, pg_notify, + pg_replslot, pg_serial, + pg_snapshots, pg_stat_tmp, and + pg_subtrans are copied as empty directories (even if + they are symbolic links). Files other than regular files and directories, such as symbolic - links and special device files, are skipped. (Symbolic links + links (other than for the directories listed above) and special + device files, are skipped. (Symbolic links in pg_tblspc are maintained.) diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 9f1eae12d8..fe557ed002 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -610,10 +610,12 @@ PostgreSQL documentation The backup will include all files in the data directory and tablespaces, including the configuration files and any additional files placed in the - directory by third parties. But only regular files and directories are - copied. Symbolic links (other than those used for tablespaces) and special - device files are skipped. (See for - the precise details.) + directory by third parties, except certain temporary files managed by + PostgreSQL. But only regular files and directories are copied, except that + symbolic links used for tablespaces are preserved. Symbolic links pointing + to certain directories known to PostgreSQL are copied as empty directories. + Other symbolic links and special device files are skipped. + See for the precise details. diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index da9b7a6f0d..1eabaef492 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -30,6 +30,7 @@ #include "replication/basebackup.h" #include "replication/walsender.h" #include "replication/walsender_private.h" +#include "storage/dsm_impl.h" #include "storage/fd.h" #include "storage/ipc.h" #include "utils/builtins.h" @@ -55,8 +56,10 @@ static int64 sendDir(char *path, int basepathlen, bool sizeonly, static bool sendFile(char *readfilename, char *tarfilename, struct stat * statbuf, bool missing_ok); static void sendFileWithContent(const char *filename, const char *content); -static void _tarWriteHeader(const char *filename, const char *linktarget, - struct stat * statbuf); +static int64 _tarWriteHeader(const char *filename, const char *linktarget, + struct stat * statbuf, bool sizeonly); +static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, + bool sizeonly); static void send_int8_string(StringInfoData *buf, int64 intval); static void SendBackupHeader(List *tablespaces); static void base_backup_cleanup(int code, Datum arg); @@ -94,6 +97,73 @@ static int64 elapsed_min_unit; /* The last check of the transfer rate. */ static int64 throttled_last; +/* + * The contents of these directories are removed or recreated during server + * start so they are not included in backups. The directories themselves are + * kept and included as empty to preserve access permissions. + */ +static const char *excludeDirContents[] = +{ + /* + * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even + * when stats_temp_directory is set because PGSS_TEXT_FILE is always created + * there. + */ + PG_STAT_TMP_DIR, + + /* + * It is generally not useful to backup the contents of this directory even + * if the intention is to restore to another master. See backup.sgml for a + * more detailed description. + */ + "pg_replslot", + + /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ + PG_DYNSHMEM_DIR, + + /* Contents removed on startup, see AsyncShmemInit(). */ + "pg_notify", + + /* + * Old contents are loaded for possible debugging but are not required for + * normal operation, see OldSerXidInit(). + */ + "pg_serial", + + /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */ + "pg_snapshots", + + /* Contents zeroed on startup, see StartupSUBTRANS(). */ + "pg_subtrans", + + /* end of list */ + NULL +}; + +/* + * List of files excluded from backups. + */ +static const char *excludeFiles[] = +{ + /* Skip auto conf temporary file. */ + PG_AUTOCONF_FILENAME ".tmp", + + /* + * If there's a backup_label or tablespace_map file, it belongs to a + * backup started by the user with pg_start_backup(). It is *not* correct + * for this backup. Our backup_label/tablespace_map is injected into the + * tar separately. + */ + BACKUP_LABEL_FILE, + TABLESPACE_MAP, + + "postmaster.pid", + "postmaster.opts", + + /* end of list */ + NULL +}; + /* * Called when ERROR or FATAL happens in perform_base_backup() after * we have started the backup - make sure we end it! @@ -415,7 +485,7 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir) } /* send the WAL file itself */ - _tarWriteHeader(pathbuf, NULL, &statbuf); + _tarWriteHeader(pathbuf, NULL, &statbuf, false); while ((cnt = fread(buf, 1, Min(sizeof(buf), XLogSegSize - len), fp)) > 0) { @@ -807,7 +877,7 @@ sendFileWithContent(const char *filename, const char *content) statbuf.st_mode = S_IRUSR | S_IWUSR; statbuf.st_size = len; - _tarWriteHeader(filename, NULL, &statbuf); + _tarWriteHeader(filename, NULL, &statbuf, false); /* Send the contents as a CopyData message */ pq_putmessage('d', content, len); @@ -858,9 +928,9 @@ sendTablespace(char *path, bool sizeonly) /* If the tablespace went away while scanning, it's no error. */ return 0; } - if (!sizeonly) - _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf); - size = 512; /* Size of the header just added */ + + size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf, + sizeonly); /* Send all the files in the tablespace version directory */ size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true); @@ -893,6 +963,9 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, dir = AllocateDir(path); while ((de = ReadDir(dir, path)) != NULL) { + int excludeIdx; + bool excludeFound; + /* Skip special stuff */ if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; @@ -903,24 +976,6 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, strlen(PG_TEMP_FILE_PREFIX)) == 0) continue; - /* skip auto conf temporary file */ - if (strncmp(de->d_name, - PG_AUTOCONF_FILENAME ".tmp", - sizeof(PG_AUTOCONF_FILENAME) + 4) == 0) - continue; - - /* - * If there's a backup_label or tablespace_map file, it belongs to a - * backup started by the user with pg_start_backup(). It is *not* - * correct for this backup, our backup_label/tablespace_map is - * injected into the tar separately. - */ - if (strcmp(de->d_name, BACKUP_LABEL_FILE) == 0) - continue; - - if (strcmp(de->d_name, TABLESPACE_MAP) == 0) - continue; - /* * Check if the postmaster has signaled us to exit, and abort with an * error in that case. The error handler further up will call @@ -938,13 +993,23 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, "and should not be used. " "Try taking another online backup."))); - snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name); + /* Scan for files that should be excluded */ + excludeFound = false; + for (excludeIdx = 0; excludeFiles[excludeIdx] != NULL; excludeIdx++) + { + if (strcmp(de->d_name, excludeFiles[excludeIdx]) == 0) + { + elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name); + excludeFound = true; + break; + } + } - /* Skip postmaster.pid and postmaster.opts in the data directory */ - if (strcmp(pathbuf, "./postmaster.pid") == 0 || - strcmp(pathbuf, "./postmaster.opts") == 0) + if (excludeFound) continue; + snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name); + /* Skip pg_control here to back up it last */ if (strcmp(pathbuf, "./global/pg_control") == 0) continue; @@ -957,33 +1022,34 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, errmsg("could not stat file or directory \"%s\": %m", pathbuf))); - /* If the file went away while scanning, it's no error. */ + /* If the file went away while scanning, it's not an error. */ continue; } - /* - * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped - * even when stats_temp_directory is set because PGSS_TEXT_FILE is - * always created there. - */ - if ((statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0) || - strncmp(de->d_name, PG_STAT_TMP_DIR, strlen(PG_STAT_TMP_DIR)) == 0) + /* Scan for directories whose contents should be excluded */ + excludeFound = false; + for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++) { - if (!sizeonly) - _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); - size += 512; - continue; + if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0) + { + elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name); + size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly); + excludeFound = true; + break; + } } + if (excludeFound) + continue; + /* - * Skip pg_replslot, not useful to copy. But include it as an empty - * directory anyway, so we get permissions right. + * Exclude contents of directory specified by statrelpath if not set + * to the default (pg_stat_tmp) which is caught in the loop above. */ - if (strcmp(de->d_name, "pg_replslot") == 0) + if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0) { - if (!sizeonly) - _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); - size += 512; /* Size of the header just added */ + elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath); + size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly); continue; } @@ -994,26 +1060,15 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, */ if (strcmp(pathbuf, "./pg_xlog") == 0) { - if (!sizeonly) - { - /* If pg_xlog is a symlink, write it as a directory anyway */ -#ifndef WIN32 - if (S_ISLNK(statbuf.st_mode)) -#else - if (pgwin32_is_junction(pathbuf)) -#endif - statbuf.st_mode = S_IFDIR | S_IRWXU; - _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); - } - size += 512; /* Size of the header just added */ + /* If pg_xlog is a symlink, write it as a directory anyway */ + size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly); /* * Also send archive_status directory (by hackishly reusing * statbuf from above ...). */ - if (!sizeonly) - _tarWriteHeader("./pg_xlog/archive_status", NULL, &statbuf); - size += 512; /* Size of the header just added */ + size += _tarWriteHeader("./pg_xlog/archive_status", NULL, &statbuf, + sizeonly); continue; /* don't recurse into pg_xlog */ } @@ -1044,9 +1099,8 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, pathbuf))); linkpath[rllen] = '\0'; - if (!sizeonly) - _tarWriteHeader(pathbuf + basepathlen + 1, linkpath, &statbuf); - size += 512; /* Size of the header just added */ + size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath, + &statbuf, sizeonly); #else /* @@ -1069,9 +1123,8 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, * Store a directory entry in the tar file so we can get the * permissions right. */ - if (!sizeonly) - _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); - size += 512; /* Size of the header just added */ + size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf, + sizeonly); /* * Call ourselves recursively for a directory, unless it happens @@ -1162,7 +1215,7 @@ sendFile(char *readfilename, char *tarfilename, struct stat * statbuf, errmsg("could not open file \"%s\": %m", readfilename))); } - _tarWriteHeader(tarfilename, NULL, statbuf); + _tarWriteHeader(tarfilename, NULL, statbuf, false); while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0) { @@ -1215,36 +1268,61 @@ sendFile(char *readfilename, char *tarfilename, struct stat * statbuf, } -static void +static int64 _tarWriteHeader(const char *filename, const char *linktarget, - struct stat * statbuf) + struct stat * statbuf, bool sizeonly) { char h[512]; enum tarError rc; - rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size, - statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, - statbuf->st_mtime); - - switch (rc) + if (!sizeonly) { - case TAR_OK: - break; - case TAR_NAME_TOO_LONG: - ereport(ERROR, - (errmsg("file name too long for tar format: \"%s\"", - filename))); - break; - case TAR_SYMLINK_TOO_LONG: - ereport(ERROR, - (errmsg("symbolic link target too long for tar format: file name \"%s\", target \"%s\"", - filename, linktarget))); - break; - default: - elog(ERROR, "unrecognized tar error: %d", rc); + rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size, + statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, + statbuf->st_mtime); + + switch (rc) + { + case TAR_OK: + break; + case TAR_NAME_TOO_LONG: + ereport(ERROR, + (errmsg("file name too long for tar format: \"%s\"", + filename))); + break; + case TAR_SYMLINK_TOO_LONG: + ereport(ERROR, + (errmsg("symbolic link target too long for tar format: " + "file name \"%s\", target \"%s\"", + filename, linktarget))); + break; + default: + elog(ERROR, "unrecognized tar error: %d", rc); + } + + pq_putmessage('d', h, sizeof(h)); } - pq_putmessage('d', h, 512); + return sizeof(h); +} + +/* + * Write tar header for a directory. If the entry in statbuf is a link then + * write it as a directory anyway. + */ +static int64 +_tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, + bool sizeonly) +{ + /* If symlink, write it as a directory anyway */ +#ifndef WIN32 + if (S_ISLNK(statbuf->st_mode)) +#else + if (pgwin32_is_junction(pathbuf)) +#endif + statbuf->st_mode = S_IFDIR | S_IRWXU; + + return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly); } /* diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index fd9857d67b..a52bd4e124 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -4,7 +4,7 @@ use Config; use PostgresNode; use TestLib; -use Test::More tests => 54; +use Test::More tests => 67; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -55,15 +55,43 @@ close CONF; $node->restart; +# Write some files to test that they are not copied. +foreach my $filename (qw(backup_label tablespace_map postgresql.auto.conf.tmp)) +{ + open FILE, ">>$pgdata/$filename"; + print FILE "DONOTCOPY"; + close FILE; +} + $node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backup" ], 'pg_basebackup runs'); ok(-f "$tempdir/backup/PG_VERSION", 'backup was created'); +# Only archive_status directory should be copied in pg_xlog/. is_deeply( [ sort(slurp_dir("$tempdir/backup/pg_xlog/")) ], [ sort qw(. .. archive_status) ], 'no WAL files copied'); +# Contents of these directories should not be copied. +foreach my $dirname (qw(pg_dynshmem pg_notify pg_replslot pg_serial pg_snapshots pg_stat_tmp pg_subtrans)) +{ + is_deeply( + [ sort(slurp_dir("$tempdir/backup/$dirname/")) ], + [ sort qw(. ..) ], + "contents of $dirname/ not copied"); +} + +# These files should not be copied. +foreach my $filename (qw(postgresql.auto.conf.tmp postmaster.opts postmaster.pid tablespace_map)) +{ + ok(! -f "$tempdir/backup/$filename", "$filename not copied"); +} + +# Make sure existing backup_label was ignored. +isnt(slurp_file("$tempdir/backup/backup_label"), 'DONOTCOPY', + 'existing backup_label not copied'); + $node->command_ok( [ 'pg_basebackup', '-D', "$tempdir/backup2", '--xlogdir', "$tempdir/xlog2" ], @@ -110,7 +138,17 @@ # skip on Windows. SKIP: { - skip "symlinks not supported on Windows", 10 if ($windows_os); + skip "symlinks not supported on Windows", 11 if ($windows_os); + + # Move pg_replslot out of $pgdata and create a symlink to it. + $node->stop; + + rename("$pgdata/pg_replslot", "$tempdir/pg_replslot") + or BAIL_OUT "could not move $pgdata/pg_replslot"; + symlink("$tempdir/pg_replslot", "$pgdata/pg_replslot") + or BAIL_OUT "could not symlink to $pgdata/pg_replslot"; + + $node->start; # Create a temporary directory in the system location and symlink it # to our physical temp location. That way we can use shorter names @@ -148,6 +186,8 @@ "tablespace symlink was updated"); closedir $dh; + ok(-d "$tempdir/backup1/pg_replslot", 'pg_replslot symlink copied as directory'); + mkdir "$tempdir/tbl=spc2"; $node->safe_psql('postgres', "DROP TABLE test1;"); $node->safe_psql('postgres', "DROP TABLESPACE tblspc1;"); From 6e654546fb61f62cc982d0c8f62241b3b30e7ef8 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 29 Sep 2016 13:16:30 +0300 Subject: [PATCH 237/871] Don't bother to lock bufmgr partitions in pg_buffercache. That makes the view a lot less disruptive to use on a production system. Without the locks, you don't get a consistent snapshot across all buffers, but that's OK. It wasn't a very useful guarantee in practice. Ivan Kartyshov, reviewed by Tomas Vondra and Robert Haas. Discusssion: --- contrib/pg_buffercache/pg_buffercache_pages.c | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index da13bde359..8bebf2384d 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -135,18 +135,13 @@ pg_buffercache_pages(PG_FUNCTION_ARGS) /* Return to original context when allocating transient memory */ MemoryContextSwitchTo(oldcontext); - /* - * To get a consistent picture of the buffer state, we must lock all - * partitions of the buffer map. Needless to say, this is horrible - * for concurrency. Must grab locks in increasing order to avoid - * possible deadlocks. - */ - for (i = 0; i < NUM_BUFFER_PARTITIONS; i++) - LWLockAcquire(BufMappingPartitionLockByIndex(i), LW_SHARED); - /* * Scan through all the buffers, saving the relevant fields in the * fctx->record structure. + * + * We don't hold the partition locks, so we don't get a consistent + * snapshot across all buffers, but we do grab the buffer header + * locks, so the information of each buffer is self-consistent. */ for (i = 0; i < NBuffers; i++) { @@ -179,16 +174,6 @@ pg_buffercache_pages(PG_FUNCTION_ARGS) UnlockBufHdr(bufHdr, buf_state); } - - /* - * And release locks. We do this in reverse order for two reasons: - * (1) Anyone else who needs more than one of the locks will be trying - * to lock them in increasing order; we don't want to release the - * other process until it can get all the locks it needs. (2) This - * avoids O(N^2) behavior inside LWLockRelease. - */ - for (i = NUM_BUFFER_PARTITIONS; --i >= 0;) - LWLockRelease(BufMappingPartitionLockByIndex(i)); } funcctx = SRF_PERCALL_SETUP(); From bf5bb2e85b6492c7245b9446efaf43d52a98db13 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 29 Sep 2016 12:00:00 -0400 Subject: [PATCH 238/871] Move fsync routines of initdb into src/common/ The intention is to used those in other utilities such as pg_basebackup and pg_receivexlog. From: Michael Paquier --- src/bin/initdb/initdb.c | 270 ++----------------------------- src/bin/initdb/nls.mk | 2 +- src/bin/pg_basebackup/nls.mk | 2 +- src/common/Makefile | 2 +- src/common/file_utils.c | 276 ++++++++++++++++++++++++++++++++ src/include/common/file_utils.h | 22 +++ src/tools/msvc/Mkvcbuild.pm | 2 +- 7 files changed, 313 insertions(+), 263 deletions(-) create mode 100644 src/common/file_utils.c create mode 100644 src/include/common/file_utils.h diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 3350e13059..e52e67df61 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -61,6 +61,7 @@ #endif #include "catalog/catalog.h" +#include "common/file_utils.h" #include "common/restricted_token.h" #include "common/username.h" #include "mb/pg_wchar.h" @@ -70,13 +71,6 @@ #include "fe_utils/string_utils.h" -/* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */ -#if defined(HAVE_SYNC_FILE_RANGE) -#define PG_FLUSH_DATA_WORKS 1 -#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) -#define PG_FLUSH_DATA_WORKS 1 -#endif - /* Ideally this would be in a .h file, but it hardly seems worth the trouble */ extern const char *select_default_timezone(const char *share_path); @@ -237,13 +231,6 @@ static char **filter_lines_with_token(char **lines, const char *token); #endif static char **readfile(const char *path); static void writefile(char *path, char **lines); -static void walkdir(const char *path, - void (*action) (const char *fname, bool isdir), - bool process_symlinks); -#ifdef PG_FLUSH_DATA_WORKS -static void pre_sync_fname(const char *fname, bool isdir); -#endif -static void fsync_fname_ext(const char *fname, bool isdir); static FILE *popen_check(const char *command, const char *mode); static void exit_nicely(void); static char *get_id(void); @@ -270,7 +257,6 @@ static void load_plpgsql(FILE *cmdfd); static void vacuum_db(FILE *cmdfd); static void make_template0(FILE *cmdfd); static void make_postgres(FILE *cmdfd); -static void fsync_pgdata(void); static void trapsig(int signum); static void check_ok(void); static char *escape_quotes(const char *src); @@ -528,177 +514,6 @@ writefile(char *path, char **lines) } } -/* - * walkdir: recursively walk a directory, applying the action to each - * regular file and directory (including the named directory itself). - * - * If process_symlinks is true, the action and recursion are also applied - * to regular files and directories that are pointed to by symlinks in the - * given directory; otherwise symlinks are ignored. Symlinks are always - * ignored in subdirectories, ie we intentionally don't pass down the - * process_symlinks flag to recursive calls. - * - * Errors are reported but not considered fatal. - * - * See also walkdir in fd.c, which is a backend version of this logic. - */ -static void -walkdir(const char *path, - void (*action) (const char *fname, bool isdir), - bool process_symlinks) -{ - DIR *dir; - struct dirent *de; - - dir = opendir(path); - if (dir == NULL) - { - fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), - progname, path, strerror(errno)); - return; - } - - while (errno = 0, (de = readdir(dir)) != NULL) - { - char subpath[MAXPGPATH]; - struct stat fst; - int sret; - - if (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0) - continue; - - snprintf(subpath, MAXPGPATH, "%s/%s", path, de->d_name); - - if (process_symlinks) - sret = stat(subpath, &fst); - else - sret = lstat(subpath, &fst); - - if (sret < 0) - { - fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), - progname, subpath, strerror(errno)); - continue; - } - - if (S_ISREG(fst.st_mode)) - (*action) (subpath, false); - else if (S_ISDIR(fst.st_mode)) - walkdir(subpath, action, false); - } - - if (errno) - fprintf(stderr, _("%s: could not read directory \"%s\": %s\n"), - progname, path, strerror(errno)); - - (void) closedir(dir); - - /* - * It's important to fsync the destination directory itself as individual - * file fsyncs don't guarantee that the directory entry for the file is - * synced. Recent versions of ext4 have made the window much wider but - * it's been an issue for ext3 and other filesystems in the past. - */ - (*action) (path, true); -} - -/* - * Hint to the OS that it should get ready to fsync() this file. - * - * Ignores errors trying to open unreadable files, and reports other errors - * non-fatally. - */ -#ifdef PG_FLUSH_DATA_WORKS - -static void -pre_sync_fname(const char *fname, bool isdir) -{ - int fd; - - fd = open(fname, O_RDONLY | PG_BINARY); - - if (fd < 0) - { - if (errno == EACCES || (isdir && errno == EISDIR)) - return; - fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), - progname, fname, strerror(errno)); - return; - } - - /* - * We do what pg_flush_data() would do in the backend: prefer to use - * sync_file_range, but fall back to posix_fadvise. We ignore errors - * because this is only a hint. - */ -#if defined(HAVE_SYNC_FILE_RANGE) - (void) sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); -#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) - (void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); -#else -#error PG_FLUSH_DATA_WORKS should not have been defined -#endif - - (void) close(fd); -} - -#endif /* PG_FLUSH_DATA_WORKS */ - -/* - * fsync_fname_ext -- Try to fsync a file or directory - * - * Ignores errors trying to open unreadable files, or trying to fsync - * directories on systems where that isn't allowed/required. Reports - * other errors non-fatally. - */ -static void -fsync_fname_ext(const char *fname, bool isdir) -{ - int fd; - int flags; - int returncode; - - /* - * Some OSs require directories to be opened read-only whereas other - * systems don't allow us to fsync files opened read-only; so we need both - * cases here. Using O_RDWR will cause us to fail to fsync files that are - * not writable by our userid, but we assume that's OK. - */ - flags = PG_BINARY; - if (!isdir) - flags |= O_RDWR; - else - flags |= O_RDONLY; - - /* - * Open the file, silently ignoring errors about unreadable files (or - * unsupported operations, e.g. opening a directory under Windows), and - * logging others. - */ - fd = open(fname, flags); - if (fd < 0) - { - if (errno == EACCES || (isdir && errno == EISDIR)) - return; - fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), - progname, fname, strerror(errno)); - return; - } - - returncode = fsync(fd); - - /* - * Some OSes don't allow us to fsync directories at all, so we can ignore - * those errors. Anything else needs to be reported. - */ - if (returncode != 0 && !(isdir && errno == EBADF)) - fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, fname, strerror(errno)); - - (void) close(fd); -} - /* * Open a subcommand with suitable error messaging */ @@ -2276,77 +2091,6 @@ make_postgres(FILE *cmdfd) PG_CMD_PUTS(*line); } -/* - * Issue fsync recursively on PGDATA and all its contents. - * - * We fsync regular files and directories wherever they are, but we - * follow symlinks only for pg_xlog and immediately under pg_tblspc. - * Other symlinks are presumed to point at files we're not responsible - * for fsyncing, and might not have privileges to write at all. - * - * Errors are reported but not considered fatal. - */ -static void -fsync_pgdata(void) -{ - bool xlog_is_symlink; - char pg_xlog[MAXPGPATH]; - char pg_tblspc[MAXPGPATH]; - - fputs(_("syncing data to disk ... "), stdout); - fflush(stdout); - - snprintf(pg_xlog, MAXPGPATH, "%s/pg_xlog", pg_data); - snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data); - - /* - * If pg_xlog is a symlink, we'll need to recurse into it separately, - * because the first walkdir below will ignore it. - */ - xlog_is_symlink = false; - -#ifndef WIN32 - { - struct stat st; - - if (lstat(pg_xlog, &st) < 0) - fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), - progname, pg_xlog, strerror(errno)); - else if (S_ISLNK(st.st_mode)) - xlog_is_symlink = true; - } -#else - if (pgwin32_is_junction(pg_xlog)) - xlog_is_symlink = true; -#endif - - /* - * If possible, hint to the kernel that we're soon going to fsync the data - * directory and its contents. - */ -#ifdef PG_FLUSH_DATA_WORKS - walkdir(pg_data, pre_sync_fname, false); - if (xlog_is_symlink) - walkdir(pg_xlog, pre_sync_fname, false); - walkdir(pg_tblspc, pre_sync_fname, true); -#endif - - /* - * Now we do the fsync()s in the same order. - * - * The main call ignores symlinks, so in addition to specially processing - * pg_xlog if it's a symlink, pg_tblspc has to be visited separately with - * process_symlinks = true. Note that if there are any plain directories - * in pg_tblspc, they'll get fsync'd twice. That's not an expected case - * so we don't worry about optimizing it. - */ - walkdir(pg_data, fsync_fname_ext, false); - if (xlog_is_symlink) - walkdir(pg_xlog, fsync_fname_ext, false); - walkdir(pg_tblspc, fsync_fname_ext, true); - - check_ok(); -} /* @@ -3512,7 +3256,10 @@ main(int argc, char *argv[]) exit_nicely(); } - fsync_pgdata(); + fputs(_("syncing data to disk ... "), stdout); + fflush(stdout); + fsync_pgdata(pg_data, progname); + check_ok(); return 0; } @@ -3574,7 +3321,12 @@ main(int argc, char *argv[]) initialize_data_directory(); if (do_sync) - fsync_pgdata(); + { + fputs(_("syncing data to disk ... "), stdout); + fflush(stdout); + fsync_pgdata(pg_data, progname); + check_ok(); + } else printf(_("\nSync to disk skipped.\nThe data directory might become corrupt if the operating system crashes.\n")); diff --git a/src/bin/initdb/nls.mk b/src/bin/initdb/nls.mk index 0d5368304e..7a12daa6fb 100644 --- a/src/bin/initdb/nls.mk +++ b/src/bin/initdb/nls.mk @@ -1,5 +1,5 @@ # src/bin/initdb/nls.mk CATALOG_NAME = initdb AVAIL_LANGUAGES = cs de es fr it ja ko pl pt_BR ru sv zh_CN -GETTEXT_FILES = findtimezone.c initdb.c ../../common/exec.c ../../common/fe_memutils.c ../../common/pgfnames.c ../../common/restricted_token.c ../../common/rmtree.c ../../common/username.c ../../common/wait_error.c ../../port/dirmod.c +GETTEXT_FILES = findtimezone.c initdb.c ../../common/exec.c ../../common/fe_memutils.c ../../common/file_utils.c ../../common/pgfnames.c ../../common/restricted_token.c ../../common/rmtree.c ../../common/username.c ../../common/wait_error.c ../../port/dirmod.c GETTEXT_TRIGGERS = simple_prompt diff --git a/src/bin/pg_basebackup/nls.mk b/src/bin/pg_basebackup/nls.mk index a34ca3d268..dba43b857e 100644 --- a/src/bin/pg_basebackup/nls.mk +++ b/src/bin/pg_basebackup/nls.mk @@ -1,5 +1,5 @@ # src/bin/pg_basebackup/nls.mk CATALOG_NAME = pg_basebackup AVAIL_LANGUAGES = de es fr it ko pl pt_BR ru zh_CN -GETTEXT_FILES = pg_basebackup.c pg_receivexlog.c pg_recvlogical.c receivelog.c streamutil.c ../../common/fe_memutils.c +GETTEXT_FILES = pg_basebackup.c pg_receivexlog.c pg_recvlogical.c receivelog.c streamutil.c ../../common/fe_memutils.c ../../common/file_utils.c GETTEXT_TRIGGERS = simple_prompt diff --git a/src/common/Makefile b/src/common/Makefile index a5fa649766..03dfaa19c4 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -44,7 +44,7 @@ OBJS_COMMON = config_info.o controldata_utils.o exec.o ip.o keywords.o \ md5.o pg_lzcompress.o pgfnames.o psprintf.o relpath.o rmtree.o \ string.o username.o wait_error.o -OBJS_FRONTEND = $(OBJS_COMMON) fe_memutils.o restricted_token.o +OBJS_FRONTEND = $(OBJS_COMMON) fe_memutils.o file_utils.o restricted_token.o OBJS_SRV = $(OBJS_COMMON:%.o=%_srv.o) diff --git a/src/common/file_utils.c b/src/common/file_utils.c new file mode 100644 index 0000000000..b6f62f7bf1 --- /dev/null +++ b/src/common/file_utils.c @@ -0,0 +1,276 @@ +/*------------------------------------------------------------------------- + * + * File-processing utility routines. + * + * Assorted utility functions to work on files. + * + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/common/file_utils.c + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include +#include +#include +#include + +#include "common/file_utils.h" + + +/* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */ +#if defined(HAVE_SYNC_FILE_RANGE) +#define PG_FLUSH_DATA_WORKS 1 +#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) +#define PG_FLUSH_DATA_WORKS 1 +#endif + +#ifdef PG_FLUSH_DATA_WORKS +static void pre_sync_fname(const char *fname, bool isdir, + const char *progname); +#endif +static void walkdir(const char *path, + void (*action) (const char *fname, bool isdir, const char *progname), + bool process_symlinks, const char *progname); + +/* + * Issue fsync recursively on PGDATA and all its contents. + * + * We fsync regular files and directories wherever they are, but we + * follow symlinks only for pg_xlog and immediately under pg_tblspc. + * Other symlinks are presumed to point at files we're not responsible + * for fsyncing, and might not have privileges to write at all. + * + * Errors are reported but not considered fatal. + */ +void +fsync_pgdata(const char *pg_data, const char *progname) +{ + bool xlog_is_symlink; + char pg_xlog[MAXPGPATH]; + char pg_tblspc[MAXPGPATH]; + + snprintf(pg_xlog, MAXPGPATH, "%s/pg_xlog", pg_data); + snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data); + + /* + * If pg_xlog is a symlink, we'll need to recurse into it separately, + * because the first walkdir below will ignore it. + */ + xlog_is_symlink = false; + +#ifndef WIN32 + { + struct stat st; + + if (lstat(pg_xlog, &st) < 0) + fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), + progname, pg_xlog, strerror(errno)); + else if (S_ISLNK(st.st_mode)) + xlog_is_symlink = true; + } +#else + if (pgwin32_is_junction(pg_xlog)) + xlog_is_symlink = true; +#endif + + /* + * If possible, hint to the kernel that we're soon going to fsync the data + * directory and its contents. + */ +#ifdef PG_FLUSH_DATA_WORKS + walkdir(pg_data, pre_sync_fname, false, progname); + if (xlog_is_symlink) + walkdir(pg_xlog, pre_sync_fname, false, progname); + walkdir(pg_tblspc, pre_sync_fname, true, progname); +#endif + + /* + * Now we do the fsync()s in the same order. + * + * The main call ignores symlinks, so in addition to specially processing + * pg_xlog if it's a symlink, pg_tblspc has to be visited separately with + * process_symlinks = true. Note that if there are any plain directories + * in pg_tblspc, they'll get fsync'd twice. That's not an expected case + * so we don't worry about optimizing it. + */ + walkdir(pg_data, fsync_fname, false, progname); + if (xlog_is_symlink) + walkdir(pg_xlog, fsync_fname, false, progname); + walkdir(pg_tblspc, fsync_fname, true, progname); +} + +/* + * walkdir: recursively walk a directory, applying the action to each + * regular file and directory (including the named directory itself). + * + * If process_symlinks is true, the action and recursion are also applied + * to regular files and directories that are pointed to by symlinks in the + * given directory; otherwise symlinks are ignored. Symlinks are always + * ignored in subdirectories, ie we intentionally don't pass down the + * process_symlinks flag to recursive calls. + * + * Errors are reported but not considered fatal. + * + * See also walkdir in fd.c, which is a backend version of this logic. + */ +static void +walkdir(const char *path, + void (*action) (const char *fname, bool isdir, const char *progname), + bool process_symlinks, const char *progname) +{ + DIR *dir; + struct dirent *de; + + dir = opendir(path); + if (dir == NULL) + { + fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), + progname, path, strerror(errno)); + return; + } + + while (errno = 0, (de = readdir(dir)) != NULL) + { + char subpath[MAXPGPATH]; + struct stat fst; + int sret; + + if (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0) + continue; + + snprintf(subpath, MAXPGPATH, "%s/%s", path, de->d_name); + + if (process_symlinks) + sret = stat(subpath, &fst); + else + sret = lstat(subpath, &fst); + + if (sret < 0) + { + fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), + progname, subpath, strerror(errno)); + continue; + } + + if (S_ISREG(fst.st_mode)) + (*action) (subpath, false, progname); + else if (S_ISDIR(fst.st_mode)) + walkdir(subpath, action, false, progname); + } + + if (errno) + fprintf(stderr, _("%s: could not read directory \"%s\": %s\n"), + progname, path, strerror(errno)); + + (void) closedir(dir); + + /* + * It's important to fsync the destination directory itself as individual + * file fsyncs don't guarantee that the directory entry for the file is + * synced. Recent versions of ext4 have made the window much wider but + * it's been an issue for ext3 and other filesystems in the past. + */ + (*action) (path, true, progname); +} + +/* + * Hint to the OS that it should get ready to fsync() this file. + * + * Ignores errors trying to open unreadable files, and reports other errors + * non-fatally. + */ +#ifdef PG_FLUSH_DATA_WORKS + +static void +pre_sync_fname(const char *fname, bool isdir, const char *progname) +{ + int fd; + + fd = open(fname, O_RDONLY | PG_BINARY); + + if (fd < 0) + { + if (errno == EACCES || (isdir && errno == EISDIR)) + return; + fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), + progname, fname, strerror(errno)); + return; + } + + /* + * We do what pg_flush_data() would do in the backend: prefer to use + * sync_file_range, but fall back to posix_fadvise. We ignore errors + * because this is only a hint. + */ +#if defined(HAVE_SYNC_FILE_RANGE) + (void) sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); +#elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) + (void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); +#else +#error PG_FLUSH_DATA_WORKS should not have been defined +#endif + + (void) close(fd); +} + +#endif /* PG_FLUSH_DATA_WORKS */ + +/* + * fsync_fname -- Try to fsync a file or directory + * + * Ignores errors trying to open unreadable files, or trying to fsync + * directories on systems where that isn't allowed/required. Reports + * other errors non-fatally. + */ +void +fsync_fname(const char *fname, bool isdir, const char *progname) +{ + int fd; + int flags; + int returncode; + + /* + * Some OSs require directories to be opened read-only whereas other + * systems don't allow us to fsync files opened read-only; so we need both + * cases here. Using O_RDWR will cause us to fail to fsync files that are + * not writable by our userid, but we assume that's OK. + */ + flags = PG_BINARY; + if (!isdir) + flags |= O_RDWR; + else + flags |= O_RDONLY; + + /* + * Open the file, silently ignoring errors about unreadable files (or + * unsupported operations, e.g. opening a directory under Windows), and + * logging others. + */ + fd = open(fname, flags); + if (fd < 0) + { + if (errno == EACCES || (isdir && errno == EISDIR)) + return; + fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), + progname, fname, strerror(errno)); + return; + } + + returncode = fsync(fd); + + /* + * Some OSes don't allow us to fsync directories at all, so we can ignore + * those errors. Anything else needs to be reported. + */ + if (returncode != 0 && !(isdir && errno == EBADF)) + fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), + progname, fname, strerror(errno)); + + (void) close(fd); +} diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h new file mode 100644 index 0000000000..d3794df7a0 --- /dev/null +++ b/src/include/common/file_utils.h @@ -0,0 +1,22 @@ +/*------------------------------------------------------------------------- + * + * File-processing utility routines for frontend code + * + * Assorted utility functions to work on files. + * + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/common/file_utils.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILE_UTILS_H +#define FILE_UTILS_H + +extern void fsync_fname(const char *fname, bool isdir, + const char *progname); +extern void fsync_pgdata(const char *pg_data, const char *progname); + +#endif /* FILE_UTILS_H */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 07c91d1849..5add0b8a91 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -115,7 +115,7 @@ sub mkvcbuild string.c username.c wait_error.c); our @pgcommonfrontendfiles = ( - @pgcommonallfiles, qw(fe_memutils.c + @pgcommonallfiles, qw(fe_memutils.c file_utils.c restricted_token.c)); our @pgcommonbkndfiles = @pgcommonallfiles; From bc34223bc1e2c51dff2007b3d3bd492a09b5a491 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 29 Sep 2016 12:00:00 -0400 Subject: [PATCH 239/871] pg_basebackup pg_receivexlog: Issue fsync more carefully Several places weren't careful about fsyncing in the way. See 1d4a0ab1 and 606e0f98 for details about required fsyncs. This adds a couple of functions in src/common/ that have an equivalent in the backend: durable_rename(), fsync_parent_path() From: Michael Paquier --- src/bin/pg_basebackup/pg_basebackup.c | 27 +++++++ src/bin/pg_basebackup/receivelog.c | 52 +++++++------ src/common/file_utils.c | 105 ++++++++++++++++++++++++-- src/include/common/file_utils.h | 7 +- 4 files changed, 162 insertions(+), 29 deletions(-) diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index d077544d62..cd7d095103 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -27,6 +27,7 @@ #include #endif +#include "common/file_utils.h" #include "common/string.h" #include "fe_utils/string_utils.h" #include "getopt_long.h" @@ -1196,6 +1197,10 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) if (copybuf != NULL) PQfreemem(copybuf); + + /* sync the resulting tar file, errors are not considered fatal */ + if (strcmp(basedir, "-") != 0) + (void) fsync_fname(filename, false, progname); } @@ -1472,6 +1477,11 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) if (basetablespace && writerecoveryconf) WriteRecoveryConf(); + + /* + * No data is synced here, everything is done for all tablespaces at the + * end. + */ } /* @@ -1950,6 +1960,23 @@ BaseBackup(void) PQclear(res); PQfinish(conn); + /* + * Make data persistent on disk once backup is completed. For tar + * format once syncing the parent directory is fine, each tar file + * created per tablespace has been already synced. In plain format, + * all the data of the base directory is synced, taking into account + * all the tablespaces. Errors are not considered fatal. + */ + if (format == 't') + { + if (strcmp(basedir, "-") != 0) + (void) fsync_fname(basedir, true, progname); + } + else + { + (void) fsync_pgdata(basedir, progname); + } + if (verbose) fprintf(stderr, "%s: base backup completed\n", progname); } diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index 3a921ebf2d..6b78a60f27 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -26,6 +26,7 @@ #include "libpq-fe.h" #include "access/xlog_internal.h" +#include "common/file_utils.h" /* fd and filename for currently open WAL file */ @@ -71,17 +72,13 @@ mark_file_as_archived(const char *basedir, const char *fname) return false; } - if (fsync(fd) != 0) - { - fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); - - close(fd); + close(fd); + if (fsync_fname(tmppath, false, progname) != 0) return false; - } - close(fd); + if (fsync_parent_path(tmppath, progname) != 0) + return false; return true; } @@ -132,6 +129,16 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) { /* File is open and ready to use */ walfile = f; + + /* + * fsync, in case of a previous crash between padding and fsyncing the + * file. + */ + if (fsync_fname(fn, false, progname) != 0) + return false; + if (fsync_parent_path(fn, progname) != 0) + return false; + return true; } if (statbuf.st_size != 0) @@ -160,6 +167,17 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) } free(zerobuf); + /* + * fsync WAL file and containing directory, to ensure the file is + * persistently created and zeroed. That's particularly important when + * using synchronous mode, where the file is modified and fsynced + * in-place, without a directory fsync. + */ + if (fsync_fname(fn, false, progname) != 0) + return false; + if (fsync_parent_path(fn, progname) != 0) + return false; + if (lseek(f, SEEK_SET, 0) != 0) { fprintf(stderr, @@ -220,10 +238,9 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) snprintf(oldfn, sizeof(oldfn), "%s/%s%s", stream->basedir, current_walfile_name, stream->partial_suffix); snprintf(newfn, sizeof(newfn), "%s/%s", stream->basedir, current_walfile_name); - if (rename(oldfn, newfn) != 0) + if (durable_rename(oldfn, newfn, progname) != 0) { - fprintf(stderr, _("%s: could not rename file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); + /* durable_rename produced a log entry */ return false; } } @@ -341,14 +358,6 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) return false; } - if (fsync(fd) != 0) - { - close(fd); - fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); - return false; - } - if (close(fd) != 0) { fprintf(stderr, _("%s: could not close file \"%s\": %s\n"), @@ -359,10 +368,9 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) /* * Now move the completed history file into place with its final name. */ - if (rename(tmppath, path) < 0) + if (durable_rename(tmppath, path, progname) < 0) { - fprintf(stderr, _("%s: could not rename file \"%s\" to \"%s\": %s\n"), - progname, tmppath, path, strerror(errno)); + /* durable_rename produced a log entry */ return false; } diff --git a/src/common/file_utils.c b/src/common/file_utils.c index b6f62f7bf1..04cd365e76 100644 --- a/src/common/file_utils.c +++ b/src/common/file_utils.c @@ -34,7 +34,7 @@ static void pre_sync_fname(const char *fname, bool isdir, const char *progname); #endif static void walkdir(const char *path, - void (*action) (const char *fname, bool isdir, const char *progname), + int (*action) (const char *fname, bool isdir, const char *progname), bool process_symlinks, const char *progname); /* @@ -120,7 +120,7 @@ fsync_pgdata(const char *pg_data, const char *progname) */ static void walkdir(const char *path, - void (*action) (const char *fname, bool isdir, const char *progname), + int (*action) (const char *fname, bool isdir, const char *progname), bool process_symlinks, const char *progname) { DIR *dir; @@ -228,7 +228,7 @@ pre_sync_fname(const char *fname, bool isdir, const char *progname) * directories on systems where that isn't allowed/required. Reports * other errors non-fatally. */ -void +int fsync_fname(const char *fname, bool isdir, const char *progname) { int fd; @@ -256,10 +256,10 @@ fsync_fname(const char *fname, bool isdir, const char *progname) if (fd < 0) { if (errno == EACCES || (isdir && errno == EISDIR)) - return; + return 0; fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), progname, fname, strerror(errno)); - return; + return -1; } returncode = fsync(fd); @@ -269,8 +269,103 @@ fsync_fname(const char *fname, bool isdir, const char *progname) * those errors. Anything else needs to be reported. */ if (returncode != 0 && !(isdir && errno == EBADF)) + { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, fname, strerror(errno)); + return -1; + } (void) close(fd); + return 0; +} + +/* + * fsync_parent_path -- fsync the parent path of a file or directory + * + * This is aimed at making file operations persistent on disk in case of + * an OS crash or power failure. + */ +int +fsync_parent_path(const char *fname, const char *progname) +{ + char parentpath[MAXPGPATH]; + + strlcpy(parentpath, fname, MAXPGPATH); + get_parent_directory(parentpath); + + /* + * get_parent_directory() returns an empty string if the input argument is + * just a file name (see comments in path.c), so handle that as being the + * current directory. + */ + if (strlen(parentpath) == 0) + strlcpy(parentpath, ".", MAXPGPATH); + + if (fsync_fname(parentpath, true, progname) != 0) + return -1; + + return 0; +} + +/* + * durable_rename -- rename(2) wrapper, issuing fsyncs required for durability + * + * Wrapper around rename, similar to the backend version. + */ +int +durable_rename(const char *oldfile, const char *newfile, const char *progname) +{ + int fd; + + /* + * First fsync the old and target path (if it exists), to ensure that they + * are properly persistent on disk. Syncing the target file is not + * strictly necessary, but it makes it easier to reason about crashes; + * because it's then guaranteed that either source or target file exists + * after a crash. + */ + if (fsync_fname(oldfile, false, progname) != 0) + return -1; + + fd = open(newfile, PG_BINARY | O_RDWR, 0); + if (fd < 0) + { + if (errno != ENOENT) + { + fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), + progname, newfile, strerror(errno)); + return -1; + } + } + else + { + if (fsync(fd) != 0) + { + fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), + progname, newfile, strerror(errno)); + close(fd); + return -1; + } + close(fd); + } + + /* Time to do the real deal... */ + if (rename(oldfile, newfile) != 0) + { + fprintf(stderr, _("%s: could not rename file \"%s\" to \"%s\": %s\n"), + progname, oldfile, newfile, strerror(errno)); + return -1; + } + + /* + * To guarantee renaming the file is persistent, fsync the file with its + * new name, and its containing directory. + */ + if (fsync_fname(newfile, false, progname) != 0) + return -1; + + if (fsync_parent_path(newfile, progname) != 0) + return -1; + + return 0; } diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index d3794df7a0..1cb263d9e2 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -15,8 +15,11 @@ #ifndef FILE_UTILS_H #define FILE_UTILS_H -extern void fsync_fname(const char *fname, bool isdir, - const char *progname); +extern int fsync_fname(const char *fname, bool isdir, + const char *progname); extern void fsync_pgdata(const char *pg_data, const char *progname); +extern int durable_rename(const char *oldfile, const char *newfile, + const char *progname); +extern int fsync_parent_path(const char *fname, const char *progname); #endif /* FILE_UTILS_H */ From 6ed2d8584cc680a2d6898480de74a57cd96176b5 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 29 Sep 2016 12:00:00 -0400 Subject: [PATCH 240/871] pg_basebackup: Add --nosync option This is useful for testing, similar to initdb's --nosync. From: Michael Paquier --- doc/src/sgml/ref/pg_basebackup.sgml | 15 ++++++++++++ src/bin/pg_basebackup/pg_basebackup.c | 28 ++++++++++++++------- src/bin/pg_basebackup/pg_receivexlog.c | 1 + src/bin/pg_basebackup/receivelog.c | 34 ++++++++++++++------------ src/bin/pg_basebackup/receivelog.h | 4 ++- 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index fe557ed002..55e913f70d 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -438,6 +438,21 @@ PostgreSQL documentation + + + + + + By default, pg_basebackup will wait for all files + to be written safely to disk. This option causes + pg_basebackup to return without waiting, which is + faster, but means that a subsequent operating system crash can leave + the base backup corrupt. Generally, this option is useful for testing + but should not be used when creating a production installation. + + + + diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index cd7d095103..0f5d9d6a87 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -69,6 +69,7 @@ static bool includewal = false; static bool streamwal = false; static bool fastcheckpoint = false; static bool writerecoveryconf = false; +static bool do_sync = true; static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static pg_time_t last_progress_report = 0; static int32 maxrate = 0; /* no limit by default */ @@ -329,6 +330,7 @@ usage(void) " set fast or spread checkpointing\n")); printf(_(" -l, --label=LABEL set backup label\n")); printf(_(" -n, --noclean do not clean up after errors\n")); + printf(_(" -N, --nosync do not wait for changes to be written safely to disk\n")); printf(_(" -P, --progress show progress information\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -460,6 +462,7 @@ LogStreamerMain(logstreamer_param *param) stream.stream_stop = reached_end_position; stream.standby_message_timeout = standby_message_timeout; stream.synchronous = false; + stream.do_sync = do_sync; stream.mark_done = true; stream.basedir = param->xlogdir; stream.partial_suffix = NULL; @@ -1199,7 +1202,7 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) PQfreemem(copybuf); /* sync the resulting tar file, errors are not considered fatal */ - if (strcmp(basedir, "-") != 0) + if (do_sync && strcmp(basedir, "-") != 0) (void) fsync_fname(filename, false, progname); } @@ -1967,14 +1970,17 @@ BaseBackup(void) * all the data of the base directory is synced, taking into account * all the tablespaces. Errors are not considered fatal. */ - if (format == 't') + if (do_sync) { - if (strcmp(basedir, "-") != 0) - (void) fsync_fname(basedir, true, progname); - } - else - { - (void) fsync_pgdata(basedir, progname); + if (format == 't') + { + if (strcmp(basedir, "-") != 0) + (void) fsync_fname(basedir, true, progname); + } + else + { + (void) fsync_pgdata(basedir, progname); + } } if (verbose) @@ -2001,6 +2007,7 @@ main(int argc, char **argv) {"compress", required_argument, NULL, 'Z'}, {"label", required_argument, NULL, 'l'}, {"noclean", no_argument, NULL, 'n'}, + {"nosync", no_argument, NULL, 'N'}, {"dbname", required_argument, NULL, 'd'}, {"host", required_argument, NULL, 'h'}, {"port", required_argument, NULL, 'p'}, @@ -2037,7 +2044,7 @@ main(int argc, char **argv) atexit(cleanup_directories_atexit); - while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nzZ:d:c:h:p:U:s:S:wWvP", + while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nNzZ:d:c:h:p:U:s:S:wWvP", long_options, &option_index)) != -1) { switch (c) @@ -2115,6 +2122,9 @@ main(int argc, char **argv) case 'n': noclean = true; break; + case 'N': + do_sync = false; + break; case 'z': #ifdef HAVE_LIBZ compresslevel = Z_DEFAULT_COMPRESSION; diff --git a/src/bin/pg_basebackup/pg_receivexlog.c b/src/bin/pg_basebackup/pg_receivexlog.c index 7f7ee9dc9b..a58a251a59 100644 --- a/src/bin/pg_basebackup/pg_receivexlog.c +++ b/src/bin/pg_basebackup/pg_receivexlog.c @@ -336,6 +336,7 @@ StreamLog(void) stream.stream_stop = stop_streaming; stream.standby_message_timeout = standby_message_timeout; stream.synchronous = synchronous; + stream.do_sync = true; stream.mark_done = false; stream.basedir = basedir; stream.partial_suffix = ".partial"; diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index 6b78a60f27..8f29d19114 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -41,8 +41,8 @@ static PGresult *HandleCopyStream(PGconn *conn, StreamCtl *stream, XLogRecPtr *stoppos); static int CopyStreamPoll(PGconn *conn, long timeout_ms); static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer); -static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len, - XLogRecPtr blockpos, int64 *last_status); +static bool ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, + int len, XLogRecPtr blockpos, int64 *last_status); static bool ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, XLogRecPtr *blockpos); static PGresult *HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf, @@ -56,7 +56,7 @@ static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos, uint32 *timeline); static bool -mark_file_as_archived(const char *basedir, const char *fname) +mark_file_as_archived(const char *basedir, const char *fname, bool do_sync) { int fd; static char tmppath[MAXPGPATH]; @@ -74,10 +74,10 @@ mark_file_as_archived(const char *basedir, const char *fname) close(fd); - if (fsync_fname(tmppath, false, progname) != 0) + if (do_sync && fsync_fname(tmppath, false, progname) != 0) return false; - if (fsync_parent_path(tmppath, progname) != 0) + if (do_sync && fsync_parent_path(tmppath, progname) != 0) return false; return true; @@ -134,9 +134,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) * fsync, in case of a previous crash between padding and fsyncing the * file. */ - if (fsync_fname(fn, false, progname) != 0) + if (stream->do_sync && fsync_fname(fn, false, progname) != 0) return false; - if (fsync_parent_path(fn, progname) != 0) + if (stream->do_sync && fsync_parent_path(fn, progname) != 0) return false; return true; @@ -173,9 +173,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) * using synchronous mode, where the file is modified and fsynced * in-place, without a directory fsync. */ - if (fsync_fname(fn, false, progname) != 0) + if (stream->do_sync && fsync_fname(fn, false, progname) != 0) return false; - if (fsync_parent_path(fn, progname) != 0) + if (stream->do_sync && fsync_parent_path(fn, progname) != 0) return false; if (lseek(f, SEEK_SET, 0) != 0) @@ -212,7 +212,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) return false; } - if (fsync(walfile) != 0) + if (stream->do_sync && fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); @@ -258,7 +258,8 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) if (currpos == XLOG_SEG_SIZE && stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, current_walfile_name)) + if (!mark_file_as_archived(stream->basedir, current_walfile_name, + stream->do_sync)) return false; } @@ -378,7 +379,8 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) if (stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, histfname)) + if (!mark_file_as_archived(stream->basedir, histfname, + stream->do_sync)) return false; } @@ -836,7 +838,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, */ if (stream->synchronous && lastFlushPosition < blockpos && walfile != -1) { - if (fsync(walfile) != 0) + if (stream->do_sync && fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); @@ -890,7 +892,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, /* Check the message type. */ if (copybuf[0] == 'k') { - if (!ProcessKeepaliveMsg(conn, copybuf, r, blockpos, + if (!ProcessKeepaliveMsg(conn, stream, copybuf, r, blockpos, &last_status)) goto error; } @@ -1043,7 +1045,7 @@ CopyStreamReceive(PGconn *conn, long timeout, char **buffer) * Process the keepalive message. */ static bool -ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len, +ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, XLogRecPtr blockpos, int64 *last_status) { int pos; @@ -1079,7 +1081,7 @@ ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len, * data has been successfully replicated or not, at the normal * shutdown of the server. */ - if (fsync(walfile) != 0) + if (stream->do_sync && fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); diff --git a/src/bin/pg_basebackup/receivelog.h b/src/bin/pg_basebackup/receivelog.h index 554ff8b5b2..7a3bbc5080 100644 --- a/src/bin/pg_basebackup/receivelog.h +++ b/src/bin/pg_basebackup/receivelog.h @@ -34,8 +34,10 @@ typedef struct StreamCtl * timeline */ int standby_message_timeout; /* Send status messages this * often */ - bool synchronous; /* Flush data on write */ + bool synchronous; /* Flush immediately WAL data on write */ bool mark_done; /* Mark segment as done in generated archive */ + bool do_sync; /* Flush to disk to ensure consistent state + * of data */ stream_stop_callback stream_stop; /* Stop streaming when returns true */ From 728a3e73e96173bb0b0b2afc0da061991f9e3d5d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 29 Sep 2016 12:00:00 -0400 Subject: [PATCH 241/871] Switch pg_basebackup commands in Postgres.pm to use --nosync On slow machines, this greatly reduces the I/O pressure induced by the tests. From: Michael Paquier --- src/test/perl/PostgresNode.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index afbdb6332b..1611ac9461 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -483,7 +483,7 @@ sub backup print "# Taking pg_basebackup $backup_name from node \"$name\"\n"; TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-p', $port, - '-x'); + '-x', '--nosync'); print "# Backup finished\n"; } From 8e91e12bc3af85ba2287866669268f6825d2cc03 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 29 Sep 2016 13:32:27 -0400 Subject: [PATCH 242/871] Allow contrib/file_fdw to read from a program, like COPY FROM PROGRAM. This patch just exposes COPY's FROM PROGRAM option in contrib/file_fdw. There don't seem to be any security issues with that that are any worse than what already exist with file_fdw and COPY; as in the existing cases, only superusers are allowed to control what gets executed. A regression test case might be nice here, but choosing a 100% portable command to run is hard. (We haven't got a test for COPY FROM PROGRAM itself, either.) Corey Huinker and Adam Gomaa, reviewed by Amit Langote Discussion: --- contrib/file_fdw/file_fdw.c | 131 ++++++++++++++++-------- contrib/file_fdw/output/file_fdw.source | 6 +- doc/src/sgml/file-fdw.sgml | 66 ++++++++---- 3 files changed, 137 insertions(+), 66 deletions(-) diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index b471991318..d325f53467 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * file_fdw.c - * foreign-data wrapper for server-side flat files. + * foreign-data wrapper for server-side flat files (or programs). * * Copyright (c) 2010-2016, PostgreSQL Global Development Group * @@ -57,8 +57,9 @@ struct FileFdwOption * fileGetOptions(), which currently doesn't bother to look at user mappings. */ static const struct FileFdwOption valid_options[] = { - /* File options */ + /* Data source options */ {"filename", ForeignTableRelationId}, + {"program", ForeignTableRelationId}, /* Format options */ /* oids option is not supported */ @@ -85,10 +86,12 @@ static const struct FileFdwOption valid_options[] = { */ typedef struct FileFdwPlanState { - char *filename; /* file to read */ - List *options; /* merged COPY options, excluding filename */ + char *filename; /* file or program to read from */ + bool is_program; /* true if filename represents an OS command */ + List *options; /* merged COPY options, excluding filename and + * is_program */ BlockNumber pages; /* estimate of file's physical size */ - double ntuples; /* estimate of number of rows in file */ + double ntuples; /* estimate of number of data rows */ } FileFdwPlanState; /* @@ -96,9 +99,11 @@ typedef struct FileFdwPlanState */ typedef struct FileFdwExecutionState { - char *filename; /* file to read */ - List *options; /* merged COPY options, excluding filename */ - CopyState cstate; /* state of reading file */ + char *filename; /* file or program to read from */ + bool is_program; /* true if filename represents an OS command */ + List *options; /* merged COPY options, excluding filename and + * is_program */ + CopyState cstate; /* COPY execution state */ } FileFdwExecutionState; /* @@ -139,7 +144,9 @@ static bool fileIsForeignScanParallelSafe(PlannerInfo *root, RelOptInfo *rel, */ static bool is_valid_option(const char *option, Oid context); static void fileGetOptions(Oid foreigntableid, - char **filename, List **other_options); + char **filename, + bool *is_program, + List **other_options); static List *get_file_fdw_attribute_options(Oid relid); static bool check_selective_binary_conversion(RelOptInfo *baserel, Oid foreigntableid, @@ -196,16 +203,16 @@ file_fdw_validator(PG_FUNCTION_ARGS) /* * Only superusers are allowed to set options of a file_fdw foreign table. - * This is because the filename is one of those options, and we don't want - * non-superusers to be able to determine which file gets read. + * This is because we don't want non-superusers to be able to control + * which file gets read or which program gets executed. * * Putting this sort of permissions check in a validator is a bit of a * crock, but there doesn't seem to be any other place that can enforce * the check more cleanly. * - * Note that the valid_options[] array disallows setting filename at any - * options level other than foreign table --- otherwise there'd still be a - * security hole. + * Note that the valid_options[] array disallows setting filename and + * program at any options level other than foreign table --- otherwise + * there'd still be a security hole. */ if (catalog == ForeignTableRelationId && !superuser()) ereport(ERROR, @@ -247,11 +254,11 @@ file_fdw_validator(PG_FUNCTION_ARGS) } /* - * Separate out filename and column-specific options, since + * Separate out filename, program, and column-specific options, since * ProcessCopyOptions won't accept them. */ - - if (strcmp(def->defname, "filename") == 0) + if (strcmp(def->defname, "filename") == 0 || + strcmp(def->defname, "program") == 0) { if (filename) ereport(ERROR, @@ -296,12 +303,13 @@ file_fdw_validator(PG_FUNCTION_ARGS) ProcessCopyOptions(NULL, NULL, true, other_options); /* - * Filename option is required for file_fdw foreign tables. + * Either filename or program option is required for file_fdw foreign + * tables. */ if (catalog == ForeignTableRelationId && filename == NULL) ereport(ERROR, (errcode(ERRCODE_FDW_DYNAMIC_PARAMETER_VALUE_NEEDED), - errmsg("filename is required for file_fdw foreign tables"))); + errmsg("either filename or program is required for file_fdw foreign tables"))); PG_RETURN_VOID(); } @@ -326,12 +334,12 @@ is_valid_option(const char *option, Oid context) /* * Fetch the options for a file_fdw foreign table. * - * We have to separate out "filename" from the other options because - * it must not appear in the options list passed to the core COPY code. + * We have to separate out filename/program from the other options because + * those must not appear in the options list passed to the core COPY code. */ static void fileGetOptions(Oid foreigntableid, - char **filename, List **other_options) + char **filename, bool *is_program, List **other_options) { ForeignTable *table; ForeignServer *server; @@ -359,9 +367,11 @@ fileGetOptions(Oid foreigntableid, options = list_concat(options, get_file_fdw_attribute_options(foreigntableid)); /* - * Separate out the filename. + * Separate out the filename or program option (we assume there is only + * one). */ *filename = NULL; + *is_program = false; prev = NULL; foreach(lc, options) { @@ -373,15 +383,22 @@ fileGetOptions(Oid foreigntableid, options = list_delete_cell(options, lc, prev); break; } + else if (strcmp(def->defname, "program") == 0) + { + *filename = defGetString(def); + *is_program = true; + options = list_delete_cell(options, lc, prev); + break; + } prev = lc; } /* - * The validator should have checked that a filename was included in the - * options, but check again, just in case. + * The validator should have checked that filename or program was included + * in the options, but check again, just in case. */ if (*filename == NULL) - elog(ERROR, "filename is required for file_fdw foreign tables"); + elog(ERROR, "either filename or program is required for file_fdw foreign tables"); *other_options = options; } @@ -475,12 +492,15 @@ fileGetForeignRelSize(PlannerInfo *root, FileFdwPlanState *fdw_private; /* - * Fetch options. We only need filename at this point, but we might as - * well get everything and not need to re-fetch it later in planning. + * Fetch options. We only need filename (or program) at this point, but + * we might as well get everything and not need to re-fetch it later in + * planning. */ fdw_private = (FileFdwPlanState *) palloc(sizeof(FileFdwPlanState)); fileGetOptions(foreigntableid, - &fdw_private->filename, &fdw_private->options); + &fdw_private->filename, + &fdw_private->is_program, + &fdw_private->options); baserel->fdw_private = (void *) fdw_private; /* Estimate relation size */ @@ -583,20 +603,25 @@ static void fileExplainForeignScan(ForeignScanState *node, ExplainState *es) { char *filename; + bool is_program; List *options; - /* Fetch options --- we only need filename at this point */ + /* Fetch options --- we only need filename and is_program at this point */ fileGetOptions(RelationGetRelid(node->ss.ss_currentRelation), - &filename, &options); + &filename, &is_program, &options); - ExplainPropertyText("Foreign File", filename, es); + if (is_program) + ExplainPropertyText("Foreign Program", filename, es); + else + ExplainPropertyText("Foreign File", filename, es); /* Suppress file size if we're not showing cost details */ if (es->costs) { struct stat stat_buf; - if (stat(filename, &stat_buf) == 0) + if (!is_program && + stat(filename, &stat_buf) == 0) ExplainPropertyLong("Foreign File Size", (long) stat_buf.st_size, es); } @@ -611,6 +636,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) { ForeignScan *plan = (ForeignScan *) node->ss.ps.plan; char *filename; + bool is_program; List *options; CopyState cstate; FileFdwExecutionState *festate; @@ -623,7 +649,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) /* Fetch options of foreign table */ fileGetOptions(RelationGetRelid(node->ss.ss_currentRelation), - &filename, &options); + &filename, &is_program, &options); /* Add any options from the plan (currently only convert_selectively) */ options = list_concat(options, plan->fdw_private); @@ -635,7 +661,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) cstate = BeginCopyFrom(NULL, node->ss.ss_currentRelation, filename, - false, + is_program, NIL, options); @@ -645,6 +671,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) */ festate = (FileFdwExecutionState *) palloc(sizeof(FileFdwExecutionState)); festate->filename = filename; + festate->is_program = is_program; festate->options = options; festate->cstate = cstate; @@ -709,7 +736,7 @@ fileReScanForeignScan(ForeignScanState *node) festate->cstate = BeginCopyFrom(NULL, node->ss.ss_currentRelation, festate->filename, - false, + festate->is_program, NIL, festate->options); } @@ -738,11 +765,22 @@ fileAnalyzeForeignTable(Relation relation, BlockNumber *totalpages) { char *filename; + bool is_program; List *options; struct stat stat_buf; /* Fetch options of foreign table */ - fileGetOptions(RelationGetRelid(relation), &filename, &options); + fileGetOptions(RelationGetRelid(relation), &filename, &is_program, &options); + + /* + * If this is a program instead of a file, just return false to skip + * analyzing the table. We could run the program and collect stats on + * whatever it currently returns, but it seems likely that in such cases + * the output would be too volatile for the stats to be useful. Maybe + * there should be an option to enable doing this? + */ + if (is_program) + return false; /* * Get size of the file. (XXX if we fail here, would it be better to just @@ -769,8 +807,8 @@ fileAnalyzeForeignTable(Relation relation, /* * fileIsForeignScanParallelSafe - * Reading a file in a parallel worker should work just the same as - * reading it in the leader, so mark scans safe. + * Reading a file, or external program, in a parallel worker should work + * just the same as reading it in the leader, so mark scans safe. */ static bool fileIsForeignScanParallelSafe(PlannerInfo *root, RelOptInfo *rel, @@ -916,9 +954,10 @@ estimate_size(PlannerInfo *root, RelOptInfo *baserel, /* * Get size of the file. It might not be there at plan time, though, in - * which case we have to use a default estimate. + * which case we have to use a default estimate. We also have to fall + * back to the default if using a program as the input. */ - if (stat(fdw_private->filename, &stat_buf) < 0) + if (fdw_private->is_program || stat(fdw_private->filename, &stat_buf) < 0) stat_buf.st_size = 10 * BLCKSZ; /* @@ -1000,6 +1039,11 @@ estimate_costs(PlannerInfo *root, RelOptInfo *baserel, * that I/O costs are equivalent to a regular table file of the same size. * However, we take per-tuple CPU costs as 10x of a seqscan, to account * for the cost of parsing records. + * + * In the case of a program source, this calculation is even more divorced + * from reality, but we have no good alternative; and it's not clear that + * the numbers we produce here matter much anyway, since there's only one + * access path for the rel. */ run_cost += seq_page_cost * pages; @@ -1036,6 +1080,7 @@ file_acquire_sample_rows(Relation onerel, int elevel, bool *nulls; bool found; char *filename; + bool is_program; List *options; CopyState cstate; ErrorContextCallback errcallback; @@ -1050,12 +1095,12 @@ file_acquire_sample_rows(Relation onerel, int elevel, nulls = (bool *) palloc(tupDesc->natts * sizeof(bool)); /* Fetch options of foreign table */ - fileGetOptions(RelationGetRelid(onerel), &filename, &options); + fileGetOptions(RelationGetRelid(onerel), &filename, &is_program, &options); /* * Create CopyState from FDW options. */ - cstate = BeginCopyFrom(NULL, onerel, filename, false, NIL, options); + cstate = BeginCopyFrom(NULL, onerel, filename, is_program, NIL, options); /* * Use per-tuple memory context to prevent leak of memory used to read diff --git a/contrib/file_fdw/output/file_fdw.source b/contrib/file_fdw/output/file_fdw.source index 6fa54409b9..01e2690a82 100644 --- a/contrib/file_fdw/output/file_fdw.source +++ b/contrib/file_fdw/output/file_fdw.source @@ -76,7 +76,7 @@ CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'csv', null ' '); -- ERROR ERROR: COPY null representation cannot use newline or carriage return CREATE FOREIGN TABLE tbl () SERVER file_server; -- ERROR -ERROR: filename is required for file_fdw foreign tables +ERROR: either filename or program is required for file_fdw foreign tables CREATE FOREIGN TABLE agg_text ( a int2 CHECK (a >= 0), b float4 @@ -132,7 +132,7 @@ ERROR: invalid option "force_not_null" HINT: There are no valid options in this context. CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (force_not_null '*'); -- ERROR ERROR: invalid option "force_not_null" -HINT: Valid options in this context are: filename, format, header, delimiter, quote, escape, null, encoding +HINT: Valid options in this context are: filename, program, format, header, delimiter, quote, escape, null, encoding -- force_null is not allowed to be specified at any foreign object level: ALTER FOREIGN DATA WRAPPER file_fdw OPTIONS (ADD force_null '*'); -- ERROR ERROR: invalid option "force_null" @@ -145,7 +145,7 @@ ERROR: invalid option "force_null" HINT: There are no valid options in this context. CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (force_null '*'); -- ERROR ERROR: invalid option "force_null" -HINT: Valid options in this context are: filename, format, header, delimiter, quote, escape, null, encoding +HINT: Valid options in this context are: filename, program, format, header, delimiter, quote, escape, null, encoding -- basic query tests SELECT * FROM agg_text WHERE b > 10.0 ORDER BY a; a | b diff --git a/doc/src/sgml/file-fdw.sgml b/doc/src/sgml/file-fdw.sgml index d3b39aa120..309a303e03 100644 --- a/doc/src/sgml/file-fdw.sgml +++ b/doc/src/sgml/file-fdw.sgml @@ -10,10 +10,11 @@ The file_fdw module provides the foreign-data wrapper file_fdw, which can be used to access data - files in the server's file system. Data files must be in a format + files in the server's file system, or to execute programs on the server + and read their output. The data file or program output must be in a format that can be read by COPY FROM; see for details. - Access to such data files is currently read-only. + Access to data files is currently read-only. @@ -27,7 +28,22 @@ - Specifies the file to be read. Required. Must be an absolute path name. + Specifies the file to be read. Must be an absolute path name. + Either filename or program must be + specified, but not both. + + + + + + program + + + + Specifies the command to be executed. The standard output of this + command will be read as though COPY FROM PROGRAM were used. + Either program or filename must be + specified, but not both. @@ -37,7 +53,7 @@ - Specifies the file's format, + Specifies the data format, the same as COPY's FORMAT option. @@ -48,7 +64,7 @@ - Specifies whether the file has a header line, + Specifies whether the data has a header line, the same as COPY's HEADER option. @@ -59,7 +75,7 @@ - Specifies the file's delimiter character, + Specifies the data delimiter character, the same as COPY's DELIMITER option. @@ -70,7 +86,7 @@ - Specifies the file's quote character, + Specifies the data quote character, the same as COPY's QUOTE option. @@ -81,7 +97,7 @@ - Specifies the file's escape character, + Specifies the data escape character, the same as COPY's ESCAPE option. @@ -92,7 +108,7 @@ - Specifies the file's null string, + Specifies the data null string, the same as COPY's NULL option. @@ -103,7 +119,7 @@ - Specifies the file's encoding, + Specifies the data encoding, the same as COPY's ENCODING option. @@ -112,11 +128,11 @@ - Note that while COPY allows options such as OIDS and HEADER - to be specified without a corresponding value, the foreign data wrapper + Note that while COPY allows options such as HEADER + to be specified without a corresponding value, the foreign table option syntax requires a value to be present in all cases. To activate - COPY options normally supplied without a value, you can - instead pass the value TRUE. + COPY options typically written without a value, you can pass + the value TRUE, since all such options are Booleans. @@ -133,7 +149,7 @@ This is a Boolean option. If true, it specifies that values of the column should not be matched against the null string (that is, the - file-level null option). This has the same effect + table-level null option). This has the same effect as listing the column in COPY's FORCE_NOT_NULL option. @@ -171,14 +187,24 @@ Changing table-level options requires superuser privileges, for security - reasons: only a superuser should be able to determine which file is read. - In principle non-superusers could be allowed to change the other options, - but that's not supported at present. + reasons: only a superuser should be able to control which file is read + or which program is run. In principle non-superusers could be allowed to + change the other options, but that's not supported at present. + + + + When specifying the program option, keep in mind that the option + string is executed by the shell. If you need to pass any arguments to the + command that come from an untrusted source, you must be careful to strip or + escape any characters that might have special meaning to the shell. + For security reasons, it is best to use a fixed command string, or at least + avoid passing any user input in it. For a foreign table using file_fdw, EXPLAIN shows - the name of the file to be read. Unless COSTS OFF is + the name of the file to be read or program to be run. + For a file, unless COSTS OFF is specified, the file size (in bytes) is shown as well. @@ -186,7 +212,7 @@ Create a Foreign Table for PostgreSQL CSV Logs - One of the obvious uses for the file_fdw is to make + One of the obvious uses for file_fdw is to make the PostgreSQL activity log available as a table for querying. To do this, first you must be logging to a CSV file, which here we will call pglog.csv. First, install file_fdw From f2af8dc5ba8801c126d877a52076d441cac78646 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 29 Sep 2016 12:00:00 -0400 Subject: [PATCH 243/871] Fix compiler warnings This was missed in bf5bb2e85b6492c7245b9446efaf43d52a98db13, because the code is only visible under PG_FLUSH_DATA_WORKS. --- src/common/file_utils.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/common/file_utils.c b/src/common/file_utils.c index 04cd365e76..1d855645b9 100644 --- a/src/common/file_utils.c +++ b/src/common/file_utils.c @@ -30,7 +30,7 @@ #endif #ifdef PG_FLUSH_DATA_WORKS -static void pre_sync_fname(const char *fname, bool isdir, +static int pre_sync_fname(const char *fname, bool isdir, const char *progname); #endif static void walkdir(const char *path, @@ -187,7 +187,7 @@ walkdir(const char *path, */ #ifdef PG_FLUSH_DATA_WORKS -static void +static int pre_sync_fname(const char *fname, bool isdir, const char *progname) { int fd; @@ -197,10 +197,10 @@ pre_sync_fname(const char *fname, bool isdir, const char *progname) if (fd < 0) { if (errno == EACCES || (isdir && errno == EISDIR)) - return; + return 0; fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), progname, fname, strerror(errno)); - return; + return -1; } /* @@ -217,6 +217,7 @@ pre_sync_fname(const char *fname, bool isdir, const char *progname) #endif (void) close(fd); + return 0; } #endif /* PG_FLUSH_DATA_WORKS */ From fd321a1dfd64d30bf1652ea6b39b654304f68ae4 Mon Sep 17 00:00:00 2001 From: Stephen Frost Date: Thu, 29 Sep 2016 22:13:38 -0400 Subject: [PATCH 244/871] Remove superuser checks in pgstattuple Now that we track initial privileges on extension objects and changes to those permissions, we can drop the superuser() checks from the various functions which are part of the pgstattuple extension and rely on the GRANT system to control access to those functions. Since a pg_upgrade will preserve the version of the extension which existed prior to the upgrade, we can't simply modify the existing functions but instead need to create new functions which remove the checks and update the SQL-level functions to use the new functions (and to REVOKE EXECUTE rights on those functions from PUBLIC). Thanks to Tom and Andres for adding support for extensions to follow update paths (see: 40b449a), allowing this patch to be much smaller since no new base version script needed to be included. Approach suggested by Noah. Reviewed by Michael Paquier. --- contrib/pgstattuple/Makefile | 7 +- contrib/pgstattuple/pgstatapprox.c | 39 +++++- contrib/pgstattuple/pgstatindex.c | 122 +++++++++++++++++- contrib/pgstattuple/pgstattuple--1.4--1.5.sql | 111 ++++++++++++++++ contrib/pgstattuple/pgstattuple.c | 41 ++++++ contrib/pgstattuple/pgstattuple.control | 2 +- doc/src/sgml/pgstattuple.sgml | 8 ++ 7 files changed, 315 insertions(+), 15 deletions(-) create mode 100644 contrib/pgstattuple/pgstattuple--1.4--1.5.sql diff --git a/contrib/pgstattuple/Makefile b/contrib/pgstattuple/Makefile index e732680dea..294077d4c1 100644 --- a/contrib/pgstattuple/Makefile +++ b/contrib/pgstattuple/Makefile @@ -4,9 +4,10 @@ MODULE_big = pgstattuple OBJS = pgstattuple.o pgstatindex.o pgstatapprox.o $(WIN32RES) EXTENSION = pgstattuple -DATA = pgstattuple--1.4.sql pgstattuple--1.3--1.4.sql \ - pgstattuple--1.2--1.3.sql pgstattuple--1.1--1.2.sql \ - pgstattuple--1.0--1.1.sql pgstattuple--unpackaged--1.0.sql +DATA = pgstattuple--1.4.sql pgstattuple--1.4--1.5.sql \ + pgstattuple--1.3--1.4.sql pgstattuple--1.2--1.3.sql \ + pgstattuple--1.1--1.2.sql pgstattuple--1.0--1.1.sql \ + pgstattuple--unpackaged--1.0.sql PGFILEDESC = "pgstattuple - tuple-level statistics" REGRESS = pgstattuple diff --git a/contrib/pgstattuple/pgstatapprox.c b/contrib/pgstattuple/pgstatapprox.c index a49ff543d2..f524fc4e30 100644 --- a/contrib/pgstattuple/pgstatapprox.c +++ b/contrib/pgstattuple/pgstatapprox.c @@ -29,6 +29,9 @@ #include "commands/vacuum.h" PG_FUNCTION_INFO_V1(pgstattuple_approx); +PG_FUNCTION_INFO_V1(pgstattuple_approx_v1_5); + +Datum pgstattuple_approx_internal(Oid relid, FunctionCallInfo fcinfo); typedef struct output_type { @@ -204,11 +207,42 @@ statapprox_heap(Relation rel, output_type *stat) /* * Returns estimated live/dead tuple statistics for the given relid. + * + * The superuser() check here must be kept as the library might be upgraded + * without the extension being upgraded, meaning that in pre-1.5 installations + * these functions could be called by any user. */ Datum pgstattuple_approx(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + (errmsg("must be superuser to use pgstattuple functions")))); + + PG_RETURN_DATUM(pgstattuple_approx_internal(relid, fcinfo)); +} + +/* + * As of pgstattuple version 1.5, we no longer need to check if the user + * is a superuser because we REVOKE EXECUTE on the SQL function from PUBLIC. + * Users can then grant access to it based on their policies. + * + * Otherwise identical to pgstattuple_approx (above). + */ +Datum +pgstattuple_approx_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + + PG_RETURN_DATUM(pgstattuple_approx_internal(relid, fcinfo)); +} + +Datum +pgstattuple_approx_internal(Oid relid, FunctionCallInfo fcinfo) +{ Relation rel; output_type stat = {0}; TupleDesc tupdesc; @@ -217,11 +251,6 @@ pgstattuple_approx(PG_FUNCTION_ARGS) HeapTuple ret; int i = 0; - if (!superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - (errmsg("must be superuser to use pgstattuple functions")))); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); diff --git a/contrib/pgstattuple/pgstatindex.c b/contrib/pgstattuple/pgstatindex.c index 6084589e07..d9a722ac6b 100644 --- a/contrib/pgstattuple/pgstatindex.c +++ b/contrib/pgstattuple/pgstatindex.c @@ -54,6 +54,14 @@ PG_FUNCTION_INFO_V1(pg_relpages); PG_FUNCTION_INFO_V1(pg_relpagesbyid); PG_FUNCTION_INFO_V1(pgstatginindex); +PG_FUNCTION_INFO_V1(pgstatindex_v1_5); +PG_FUNCTION_INFO_V1(pgstatindexbyid_v1_5); +PG_FUNCTION_INFO_V1(pg_relpages_v1_5); +PG_FUNCTION_INFO_V1(pg_relpagesbyid_v1_5); +PG_FUNCTION_INFO_V1(pgstatginindex_v1_5); + +Datum pgstatginindex_internal(Oid relid, FunctionCallInfo fcinfo); + #define IS_INDEX(r) ((r)->rd_rel->relkind == RELKIND_INDEX) #define IS_BTREE(r) ((r)->rd_rel->relam == BTREE_AM_OID) #define IS_GIN(r) ((r)->rd_rel->relam == GIN_AM_OID) @@ -99,6 +107,10 @@ static Datum pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo); * pgstatindex() * * Usage: SELECT * FROM pgstatindex('t1_pkey'); + * + * The superuser() check here must be kept as the library might be upgraded + * without the extension being upgraded, meaning that in pre-1.5 installations + * these functions could be called by any user. * ------------------------------------------------------ */ Datum @@ -119,6 +131,31 @@ pgstatindex(PG_FUNCTION_ARGS) PG_RETURN_DATUM(pgstatindex_impl(rel, fcinfo)); } +/* + * As of pgstattuple version 1.5, we no longer need to check if the user + * is a superuser because we REVOKE EXECUTE on the function from PUBLIC. + * Users can then grant access to it based on their policies. + * + * Otherwise identical to pgstatindex (above). + */ +Datum +pgstatindex_v1_5(PG_FUNCTION_ARGS) +{ + text *relname = PG_GETARG_TEXT_P(0); + Relation rel; + RangeVar *relrv; + + relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); + rel = relation_openrv(relrv, AccessShareLock); + + PG_RETURN_DATUM(pgstatindex_impl(rel, fcinfo)); +} + +/* + * The superuser() check here must be kept as the library might be upgraded + * without the extension being upgraded, meaning that in pre-1.5 installations + * these functions could be called by any user. + */ Datum pgstatindexbyid(PG_FUNCTION_ARGS) { @@ -135,6 +172,18 @@ pgstatindexbyid(PG_FUNCTION_ARGS) PG_RETURN_DATUM(pgstatindex_impl(rel, fcinfo)); } +/* No need for superuser checks in v1.5, see above */ +Datum +pgstatindexbyid_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + Relation rel; + + rel = relation_open(relid, AccessShareLock); + + PG_RETURN_DATUM(pgstatindex_impl(rel, fcinfo)); +} + static Datum pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo) { @@ -292,6 +341,8 @@ pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo) * * Usage: SELECT pg_relpages('t1'); * SELECT pg_relpages('t1_pkey'); + * + * Must keep superuser() check, see above. * -------------------------------------------------------- */ Datum @@ -319,6 +370,28 @@ pg_relpages(PG_FUNCTION_ARGS) PG_RETURN_INT64(relpages); } +/* No need for superuser checks in v1.5, see above */ +Datum +pg_relpages_v1_5(PG_FUNCTION_ARGS) +{ + text *relname = PG_GETARG_TEXT_P(0); + int64 relpages; + Relation rel; + RangeVar *relrv; + + relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); + rel = relation_openrv(relrv, AccessShareLock); + + /* note: this will work OK on non-local temp tables */ + + relpages = RelationGetNumberOfBlocks(rel); + + relation_close(rel, AccessShareLock); + + PG_RETURN_INT64(relpages); +} + +/* Must keep superuser() check, see above. */ Datum pg_relpagesbyid(PG_FUNCTION_ARGS) { @@ -342,16 +415,58 @@ pg_relpagesbyid(PG_FUNCTION_ARGS) PG_RETURN_INT64(relpages); } +/* No need for superuser checks in v1.5, see above */ +Datum +pg_relpagesbyid_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + int64 relpages; + Relation rel; + + rel = relation_open(relid, AccessShareLock); + + /* note: this will work OK on non-local temp tables */ + + relpages = RelationGetNumberOfBlocks(rel); + + relation_close(rel, AccessShareLock); + + PG_RETURN_INT64(relpages); +} + /* ------------------------------------------------------ * pgstatginindex() * * Usage: SELECT * FROM pgstatginindex('ginindex'); + * + * Must keep superuser() check, see above. * ------------------------------------------------------ */ Datum pgstatginindex(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + (errmsg("must be superuser to use pgstattuple functions")))); + + PG_RETURN_DATUM(pgstatginindex_internal(relid, fcinfo)); +} + +/* No need for superuser checks in v1.5, see above */ +Datum +pgstatginindex_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + + PG_RETURN_DATUM(pgstatginindex_internal(relid, fcinfo)); +} + +Datum +pgstatginindex_internal(Oid relid, FunctionCallInfo fcinfo) +{ Relation rel; Buffer buffer; Page page; @@ -363,11 +478,6 @@ pgstatginindex(PG_FUNCTION_ARGS) bool nulls[3] = {false, false, false}; Datum result; - if (!superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - (errmsg("must be superuser to use pgstattuple functions")))); - rel = relation_open(relid, AccessShareLock); if (!IS_INDEX(rel) || !IS_GIN(rel)) @@ -415,5 +525,5 @@ pgstatginindex(PG_FUNCTION_ARGS) tuple = heap_form_tuple(tupleDesc, values, nulls); result = HeapTupleGetDatum(tuple); - PG_RETURN_DATUM(result); + return (result); } diff --git a/contrib/pgstattuple/pgstattuple--1.4--1.5.sql b/contrib/pgstattuple/pgstattuple--1.4--1.5.sql new file mode 100644 index 0000000000..65d7f19c2a --- /dev/null +++ b/contrib/pgstattuple/pgstattuple--1.4--1.5.sql @@ -0,0 +1,111 @@ +/* contrib/pgstattuple/pgstattuple--1.4--1.5.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pgstattuple UPDATE TO '1.5'" to load this file. \quit + +CREATE OR REPLACE FUNCTION pgstattuple(IN relname text, + OUT table_len BIGINT, -- physical table length in bytes + OUT tuple_count BIGINT, -- number of live tuples + OUT tuple_len BIGINT, -- total tuples length in bytes + OUT tuple_percent FLOAT8, -- live tuples in % + OUT dead_tuple_count BIGINT, -- number of dead tuples + OUT dead_tuple_len BIGINT, -- total dead tuples length in bytes + OUT dead_tuple_percent FLOAT8, -- dead tuples in % + OUT free_space BIGINT, -- free space in bytes + OUT free_percent FLOAT8) -- free space in % +AS 'MODULE_PATHNAME', 'pgstattuple_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstattuple(text) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION pgstatindex(IN relname text, + OUT version INT, + OUT tree_level INT, + OUT index_size BIGINT, + OUT root_block_no BIGINT, + OUT internal_pages BIGINT, + OUT leaf_pages BIGINT, + OUT empty_pages BIGINT, + OUT deleted_pages BIGINT, + OUT avg_leaf_density FLOAT8, + OUT leaf_fragmentation FLOAT8) +AS 'MODULE_PATHNAME', 'pgstatindex_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstatindex(text) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION pg_relpages(IN relname text) +RETURNS BIGINT +AS 'MODULE_PATHNAME', 'pg_relpages_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_relpages(text) FROM PUBLIC; + +/* New stuff in 1.1 begins here */ + +CREATE OR REPLACE FUNCTION pgstatginindex(IN relname regclass, + OUT version INT4, + OUT pending_pages INT4, + OUT pending_tuples BIGINT) +AS 'MODULE_PATHNAME', 'pgstatginindex_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstatginindex(regclass) FROM PUBLIC; + +/* New stuff in 1.2 begins here */ + +CREATE OR REPLACE FUNCTION pgstattuple(IN reloid regclass, + OUT table_len BIGINT, -- physical table length in bytes + OUT tuple_count BIGINT, -- number of live tuples + OUT tuple_len BIGINT, -- total tuples length in bytes + OUT tuple_percent FLOAT8, -- live tuples in % + OUT dead_tuple_count BIGINT, -- number of dead tuples + OUT dead_tuple_len BIGINT, -- total dead tuples length in bytes + OUT dead_tuple_percent FLOAT8, -- dead tuples in % + OUT free_space BIGINT, -- free space in bytes + OUT free_percent FLOAT8) -- free space in % +AS 'MODULE_PATHNAME', 'pgstattuplebyid_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstattuple(regclass) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION pgstatindex(IN relname regclass, + OUT version INT, + OUT tree_level INT, + OUT index_size BIGINT, + OUT root_block_no BIGINT, + OUT internal_pages BIGINT, + OUT leaf_pages BIGINT, + OUT empty_pages BIGINT, + OUT deleted_pages BIGINT, + OUT avg_leaf_density FLOAT8, + OUT leaf_fragmentation FLOAT8) +AS 'MODULE_PATHNAME', 'pgstatindexbyid_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstatindex(regclass) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION pg_relpages(IN relname regclass) +RETURNS BIGINT +AS 'MODULE_PATHNAME', 'pg_relpagesbyid_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_relpages(regclass) FROM PUBLIC; + +/* New stuff in 1.3 begins here */ + +CREATE OR REPLACE FUNCTION pgstattuple_approx(IN reloid regclass, + OUT table_len BIGINT, -- physical table length in bytes + OUT scanned_percent FLOAT8, -- what percentage of the table's pages was scanned + OUT approx_tuple_count BIGINT, -- estimated number of live tuples + OUT approx_tuple_len BIGINT, -- estimated total length in bytes of live tuples + OUT approx_tuple_percent FLOAT8, -- live tuples in % (based on estimate) + OUT dead_tuple_count BIGINT, -- exact number of dead tuples + OUT dead_tuple_len BIGINT, -- exact total length in bytes of dead tuples + OUT dead_tuple_percent FLOAT8, -- dead tuples in % (based on estimate) + OUT approx_free_space BIGINT, -- estimated free space in bytes + OUT approx_free_percent FLOAT8) -- free space in % (based on estimate) +AS 'MODULE_PATHNAME', 'pgstattuple_approx_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstattuple_approx(regclass) FROM PUBLIC; diff --git a/contrib/pgstattuple/pgstattuple.c b/contrib/pgstattuple/pgstattuple.c index c1122b496a..68b07aaf26 100644 --- a/contrib/pgstattuple/pgstattuple.c +++ b/contrib/pgstattuple/pgstattuple.c @@ -40,7 +40,9 @@ PG_MODULE_MAGIC; PG_FUNCTION_INFO_V1(pgstattuple); +PG_FUNCTION_INFO_V1(pgstattuple_v1_5); PG_FUNCTION_INFO_V1(pgstattuplebyid); +PG_FUNCTION_INFO_V1(pgstattuplebyid_v1_5); /* * struct pgstattuple_type @@ -152,6 +154,10 @@ build_pgstattuple_type(pgstattuple_type *stat, FunctionCallInfo fcinfo) * * C FUNCTION definition * pgstattuple(text) returns pgstattuple_type + * + * The superuser() check here must be kept as the library might be upgraded + * without the extension being upgraded, meaning that in pre-1.5 installations + * these functions could be called by any user. * ---------- */ @@ -174,6 +180,28 @@ pgstattuple(PG_FUNCTION_ARGS) PG_RETURN_DATUM(pgstat_relation(rel, fcinfo)); } +/* + * As of pgstattuple version 1.5, we no longer need to check if the user + * is a superuser because we REVOKE EXECUTE on the function from PUBLIC. + * Users can then grant access to it based on their policies. + * + * Otherwise identical to pgstattuple (above). + */ +Datum +pgstattuple_v1_5(PG_FUNCTION_ARGS) +{ + text *relname = PG_GETARG_TEXT_P(0); + RangeVar *relrv; + Relation rel; + + /* open relation */ + relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); + rel = relation_openrv(relrv, AccessShareLock); + + PG_RETURN_DATUM(pgstat_relation(rel, fcinfo)); +} + +/* Must keep superuser() check, see above. */ Datum pgstattuplebyid(PG_FUNCTION_ARGS) { @@ -191,6 +219,19 @@ pgstattuplebyid(PG_FUNCTION_ARGS) PG_RETURN_DATUM(pgstat_relation(rel, fcinfo)); } +/* Remove superuser() check for 1.5 version, see above */ +Datum +pgstattuplebyid_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + Relation rel; + + /* open relation */ + rel = relation_open(relid, AccessShareLock); + + PG_RETURN_DATUM(pgstat_relation(rel, fcinfo)); +} + /* * pgstat_relation */ diff --git a/contrib/pgstattuple/pgstattuple.control b/contrib/pgstattuple/pgstattuple.control index fa328fd664..6af40757b2 100644 --- a/contrib/pgstattuple/pgstattuple.control +++ b/contrib/pgstattuple/pgstattuple.control @@ -1,5 +1,5 @@ # pgstattuple extension comment = 'show tuple-level statistics' -default_version = '1.4' +default_version = '1.5' module_pathname = '$libdir/pgstattuple' relocatable = true diff --git a/doc/src/sgml/pgstattuple.sgml b/doc/src/sgml/pgstattuple.sgml index 61340bedbc..9ada5d209a 100644 --- a/doc/src/sgml/pgstattuple.sgml +++ b/doc/src/sgml/pgstattuple.sgml @@ -12,6 +12,14 @@ obtain tuple-level statistics. + + As these functions return detailed page-level information, only the superuser + has EXECUTE privileges on them upon installation. After the functions have + been installed, users may issue GRANT commands to change + the privileges on the functions to allow non-superusers to execute them. See + the description of the command for specifics. + + Functions From 3d39244e6e7374febff59eba338f7a82f696a618 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Fri, 30 Sep 2016 11:19:30 +0200 Subject: [PATCH 245/871] Retry opening new segments in pg_xlogdump --folllow There is a small window between when the server closes out the existing segment and the new one is created. Put a loop around the open call in this case to make sure we wait for the new file to actually appear. --- src/bin/pg_xlogdump/pg_xlogdump.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_xlogdump/pg_xlogdump.c b/src/bin/pg_xlogdump/pg_xlogdump.c index 02575eb1c5..9ad9321e1f 100644 --- a/src/bin/pg_xlogdump/pg_xlogdump.c +++ b/src/bin/pg_xlogdump/pg_xlogdump.c @@ -249,6 +249,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id, if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo)) { char fname[MAXFNAMELEN]; + int tries; /* Switch to another logfile segment */ if (sendFile >= 0) @@ -258,7 +259,30 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id, XLogFileName(fname, timeline_id, sendSegNo); - sendFile = fuzzy_open_file(directory, fname); + /* + * In follow mode there is a short period of time after the + * server has written the end of the previous file before the + * new file is available. So we loop for 5 seconds looking + * for the file to appear before giving up. + */ + for (tries = 0; tries < 10; tries++) + { + sendFile = fuzzy_open_file(directory, fname); + if (sendFile >= 0) + break; + if (errno == ENOENT) + { + int save_errno = errno; + + /* File not there yet, try again */ + pg_usleep(500 * 1000); + + errno = save_errno; + continue; + } + /* Any other error, fall through and fail */ + break; + } if (sendFile < 0) fatal_error("could not find file \"%s\": %s", From f1a469c9f1cfeab9f9c7d4a5d3e75892e7b6f60c Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 30 Aug 2016 12:00:00 -0400 Subject: [PATCH 246/871] Fix use of offsetof() Using offsetof() with a run-time computed argument is not allowed in either C or C++. Apparently, gcc allows it, but g++ doesn't. Reviewed-by: Heikki Linnakangas Reviewed-by: Thomas Munro --- contrib/bloom/blutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/bloom/blutils.c b/contrib/bloom/blutils.c index debf4f46eb..b68a0d1d0c 100644 --- a/contrib/bloom/blutils.c +++ b/contrib/bloom/blutils.c @@ -75,7 +75,7 @@ _PG_init(void) bl_relopt_tab[i + 1].optname = MemoryContextStrdup(TopMemoryContext, buf); bl_relopt_tab[i + 1].opttype = RELOPT_TYPE_INT; - bl_relopt_tab[i + 1].offset = offsetof(BloomOptions, bitSize[i]); + bl_relopt_tab[i + 1].offset = offsetof(BloomOptions, bitSize[0]) + sizeof(int) * i; } } From 1c0cf52b39ca3a9a79661129cff918dc000a55eb Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 30 Aug 2016 12:00:00 -0400 Subject: [PATCH 247/871] Use return instead of exit() in configure Using exit() requires stdlib.h, which is not included. Use return instead. Also add return type for main(). Reviewed-by: Heikki Linnakangas Reviewed-by: Thomas Munro --- config/c-compiler.m4 | 4 +++- config/c-library.m4 | 4 +++- configure | 12 +++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 index a7f6773ae1..7d901e1f1a 100644 --- a/config/c-compiler.m4 +++ b/config/c-compiler.m4 @@ -71,8 +71,10 @@ int does_int64_work() return 0; return 1; } + +int main() { - exit(! does_int64_work()); + return (! does_int64_work()); }])], [Ac_cachevar=yes], [Ac_cachevar=no], diff --git a/config/c-library.m4 b/config/c-library.m4 index d330b0cf95..0a7452c176 100644 --- a/config/c-library.m4 +++ b/config/c-library.m4 @@ -204,8 +204,10 @@ int does_int64_snprintf_work() return 0; /* either multiply or snprintf is busted */ return 1; } + +int main() { - exit(! does_int64_snprintf_work()); + return (! does_int64_snprintf_work()); }]])], [pgac_cv_snprintf_long_long_int_modifier=$pgac_modifier; break], [], diff --git a/configure b/configure index 55c771a11e..3eb0faf77d 100755 --- a/configure +++ b/configure @@ -13594,8 +13594,10 @@ int does_int64_work() return 0; return 1; } + +int main() { - exit(! does_int64_work()); + return (! does_int64_work()); } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -13676,8 +13678,10 @@ int does_int64_work() return 0; return 1; } + +int main() { - exit(! does_int64_work()); + return (! does_int64_work()); } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -13770,8 +13774,10 @@ int does_int64_snprintf_work() return 0; /* either multiply or snprintf is busted */ return 1; } + +int main() { - exit(! does_int64_snprintf_work()); + return (! does_int64_snprintf_work()); } _ACEOF if ac_fn_c_try_run "$LINENO"; then : From a5da81359dbddfee49ea7800b43d99c204b4d31f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 30 Aug 2016 12:00:00 -0400 Subject: [PATCH 248/871] Add missing include files to configure tests atoi() needs stdlib.h strcmp() needs string.h Reviewed-by: Heikki Linnakangas Reviewed-by: Thomas Munro --- config/c-library.m4 | 4 +++- configure | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/c-library.m4 b/config/c-library.m4 index 0a7452c176..4ff8dae141 100644 --- a/config/c-library.m4 +++ b/config/c-library.m4 @@ -42,7 +42,8 @@ if test "$ac_cv_member_struct_tm_tm_zone" = yes; then fi AC_CACHE_CHECK(for tzname, ac_cv_var_tzname, [AC_LINK_IFELSE([AC_LANG_PROGRAM( -[[#include +[[#include +#include #ifndef tzname /* For SGI. */ extern char *tzname[]; /* RS6000 and others reject char **tzname. */ #endif @@ -184,6 +185,7 @@ AC_DEFUN([PGAC_FUNC_SNPRINTF_LONG_LONG_INT_MODIFIER], AC_CACHE_VAL(pgac_cv_snprintf_long_long_int_modifier, [for pgac_modifier in 'll' 'q' 'I64'; do AC_RUN_IFELSE([AC_LANG_SOURCE([[#include +#include typedef long long int ac_int64; #define INT64_FORMAT "%${pgac_modifier}d" diff --git a/configure b/configure index 3eb0faf77d..1021fd539e 100755 --- a/configure +++ b/configure @@ -11569,6 +11569,7 @@ if ${ac_cv_var_tzname+:} false; then : else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +#include #include #ifndef tzname /* For SGI. */ extern char *tzname[]; /* RS6000 and others reject char **tzname. */ @@ -13754,6 +13755,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include +#include typedef long long int ac_int64; #define INT64_FORMAT "%${pgac_modifier}d" From 0665023b4435a469e42289d7065c436967a022b6 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 30 Aug 2016 12:00:00 -0400 Subject: [PATCH 249/871] Remove unnecessary prototypes Prototypes for functions implementing V1-callable functions are no longer necessary. Reviewed-by: Heikki Linnakangas Reviewed-by: Thomas Munro --- contrib/bloom/bloom.h | 1 - contrib/btree_gist/btree_gist.h | 10 ---------- contrib/dblink/dblink.h | 24 ------------------------ contrib/isn/isn.h | 19 ------------------- contrib/pgcrypto/pgcrypto.h | 13 ------------- contrib/tablefunc/tablefunc.h | 9 --------- 6 files changed, 76 deletions(-) diff --git a/contrib/bloom/bloom.h b/contrib/bloom/bloom.h index bc451a00db..18f5127059 100644 --- a/contrib/bloom/bloom.h +++ b/contrib/bloom/bloom.h @@ -174,7 +174,6 @@ typedef BloomScanOpaqueData *BloomScanOpaque; /* blutils.c */ extern void _PG_init(void); -extern Datum blhandler(PG_FUNCTION_ARGS); extern void initBloomState(BloomState *state, Relation index); extern void BloomFillMetapage(Relation index, Page metaPage); extern void BloomInitMetapage(Relation index); diff --git a/contrib/btree_gist/btree_gist.h b/contrib/btree_gist/btree_gist.h index dcffbb5178..191202aede 100644 --- a/contrib/btree_gist/btree_gist.h +++ b/contrib/btree_gist/btree_gist.h @@ -34,14 +34,4 @@ enum gbtree_type gbt_t_inet }; - - -/* - * Generic btree functions - */ - -Datum gbtreekey_in(PG_FUNCTION_ARGS); - -Datum gbtreekey_out(PG_FUNCTION_ARGS); - #endif diff --git a/contrib/dblink/dblink.h b/contrib/dblink/dblink.h index 1b94912152..1102236967 100644 --- a/contrib/dblink/dblink.h +++ b/contrib/dblink/dblink.h @@ -36,28 +36,4 @@ #include "fmgr.h" -/* - * External declarations - */ -extern Datum dblink_connect(PG_FUNCTION_ARGS); -extern Datum dblink_disconnect(PG_FUNCTION_ARGS); -extern Datum dblink_open(PG_FUNCTION_ARGS); -extern Datum dblink_close(PG_FUNCTION_ARGS); -extern Datum dblink_fetch(PG_FUNCTION_ARGS); -extern Datum dblink_record(PG_FUNCTION_ARGS); -extern Datum dblink_send_query(PG_FUNCTION_ARGS); -extern Datum dblink_get_result(PG_FUNCTION_ARGS); -extern Datum dblink_get_connections(PG_FUNCTION_ARGS); -extern Datum dblink_is_busy(PG_FUNCTION_ARGS); -extern Datum dblink_cancel_query(PG_FUNCTION_ARGS); -extern Datum dblink_error_message(PG_FUNCTION_ARGS); -extern Datum dblink_exec(PG_FUNCTION_ARGS); -extern Datum dblink_get_pkey(PG_FUNCTION_ARGS); -extern Datum dblink_build_sql_insert(PG_FUNCTION_ARGS); -extern Datum dblink_build_sql_delete(PG_FUNCTION_ARGS); -extern Datum dblink_build_sql_update(PG_FUNCTION_ARGS); -extern Datum dblink_current_query(PG_FUNCTION_ARGS); -extern Datum dblink_get_notify(PG_FUNCTION_ARGS); -extern Datum dblink_fdw_validator(PG_FUNCTION_ARGS); - #endif /* DBLINK_H */ diff --git a/contrib/isn/isn.h b/contrib/isn/isn.h index d8291c2b06..e1a52b27d5 100644 --- a/contrib/isn/isn.h +++ b/contrib/isn/isn.h @@ -30,25 +30,6 @@ typedef uint64 ean13; #define PG_GETARG_EAN13(n) PG_GETARG_INT64(n) #define PG_RETURN_EAN13(x) PG_RETURN_INT64(x) -extern Datum isn_out(PG_FUNCTION_ARGS); -extern Datum ean13_out(PG_FUNCTION_ARGS); -extern Datum ean13_in(PG_FUNCTION_ARGS); -extern Datum isbn_in(PG_FUNCTION_ARGS); -extern Datum ismn_in(PG_FUNCTION_ARGS); -extern Datum issn_in(PG_FUNCTION_ARGS); -extern Datum upc_in(PG_FUNCTION_ARGS); - -extern Datum isbn_cast_from_ean13(PG_FUNCTION_ARGS); -extern Datum ismn_cast_from_ean13(PG_FUNCTION_ARGS); -extern Datum issn_cast_from_ean13(PG_FUNCTION_ARGS); -extern Datum upc_cast_from_ean13(PG_FUNCTION_ARGS); - -extern Datum is_valid(PG_FUNCTION_ARGS); -extern Datum make_valid(PG_FUNCTION_ARGS); - -extern Datum accept_weak_input(PG_FUNCTION_ARGS); -extern Datum weak_input_status(PG_FUNCTION_ARGS); - extern void initialize(void); #endif /* ISN_H */ diff --git a/contrib/pgcrypto/pgcrypto.h b/contrib/pgcrypto/pgcrypto.h index dfc7a10590..65a1ed3157 100644 --- a/contrib/pgcrypto/pgcrypto.h +++ b/contrib/pgcrypto/pgcrypto.h @@ -34,17 +34,4 @@ #include "fmgr.h" -/* exported functions */ -Datum pg_digest(PG_FUNCTION_ARGS); -Datum pg_hmac(PG_FUNCTION_ARGS); -Datum pg_gen_salt(PG_FUNCTION_ARGS); -Datum pg_gen_salt_rounds(PG_FUNCTION_ARGS); -Datum pg_crypt(PG_FUNCTION_ARGS); -Datum pg_encrypt(PG_FUNCTION_ARGS); -Datum pg_decrypt(PG_FUNCTION_ARGS); -Datum pg_encrypt_iv(PG_FUNCTION_ARGS); -Datum pg_decrypt_iv(PG_FUNCTION_ARGS); -Datum pg_random_bytes(PG_FUNCTION_ARGS); -Datum pg_random_uuid(PG_FUNCTION_ARGS); - #endif diff --git a/contrib/tablefunc/tablefunc.h b/contrib/tablefunc/tablefunc.h index 3477ed823f..34d44a8fa1 100644 --- a/contrib/tablefunc/tablefunc.h +++ b/contrib/tablefunc/tablefunc.h @@ -36,13 +36,4 @@ #include "fmgr.h" -/* - * External declarations - */ -extern Datum normal_rand(PG_FUNCTION_ARGS); -extern Datum crosstab(PG_FUNCTION_ARGS); -extern Datum crosstab_hash(PG_FUNCTION_ARGS); -extern Datum connectby_text(PG_FUNCTION_ARGS); -extern Datum connectby_text_serial(PG_FUNCTION_ARGS); - #endif /* TABLEFUNC_H */ From 330b48b94b53bcbbc490f952d6798d5ab637721a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 30 Sep 2016 12:00:00 -0400 Subject: [PATCH 250/871] Separate enum from struct Otherwise the enum symbols are not visible outside the struct in C++. Reviewed-by: Thomas Munro --- src/include/utils/jsonb.h | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/include/utils/jsonb.h b/src/include/utils/jsonb.h index fa52afcb5c..48ca9dc913 100644 --- a/src/include/utils/jsonb.h +++ b/src/include/utils/jsonb.h @@ -219,6 +219,20 @@ typedef struct #define JB_ROOT_IS_ARRAY(jbp_) ( *(uint32*) VARDATA(jbp_) & JB_FARRAY) +enum jbvType +{ + /* Scalar types */ + jbvNull = 0x0, + jbvString, + jbvNumeric, + jbvBool, + /* Composite types */ + jbvArray = 0x10, + jbvObject, + /* Binary (i.e. struct Jsonb) jbvArray/jbvObject */ + jbvBinary +}; + /* * JsonbValue: In-memory representation of Jsonb. This is a convenient * deserialized representation, that can easily support using the "val" @@ -227,19 +241,7 @@ typedef struct */ struct JsonbValue { - enum - { - /* Scalar types */ - jbvNull = 0x0, - jbvString, - jbvNumeric, - jbvBool, - /* Composite types */ - jbvArray = 0x10, - jbvObject, - /* Binary (i.e. struct Jsonb) jbvArray/jbvObject */ - jbvBinary - } type; /* Influences sort order */ + jbvType type; /* Influences sort order */ union { From cd03890d0b5d9cbcfca169195d71f1693f84cafa Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 30 Sep 2016 12:00:00 -0400 Subject: [PATCH 251/871] Fix breakage in previous change --- src/include/utils/jsonb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/utils/jsonb.h b/src/include/utils/jsonb.h index 48ca9dc913..470d5b1050 100644 --- a/src/include/utils/jsonb.h +++ b/src/include/utils/jsonb.h @@ -241,7 +241,7 @@ enum jbvType */ struct JsonbValue { - jbvType type; /* Influences sort order */ + enum jbvType type; /* Influences sort order */ union { From 5afcd2aa740f87fa6cec7c6693a4891b9df2b67d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 30 Sep 2016 20:39:00 -0400 Subject: [PATCH 252/871] Fix multiple portability issues in pg_upgrade's rewriteVisibilityMap(). This is new code in 9.6, and evidently we missed out testing it as thoroughly as it should have been. Bugs fixed here: 1. Use binary not text mode to open the files on Windows. Before, if the visibility map chanced to contain two bytes that looked like \r\n, Windows' read() would convert that to \n, which both corrupts the map data and causes the file to look shorter than it should. Unless you were *very* unlucky and had an exact multiple of 8K such occurrences in each VM file, this would cause pg_upgrade to report a failure, though with a rather obscure error message. 2. The code for copying rebuilt bytes into the output was simply wrong. It chanced to work okay on little-endian machines but would emit the bytes in the wrong order on big-endian, leading to silent corruption of the visibility map data. 3. The code was careless about alignment of the working buffers. Given all three of an alignment-picky architecture, a compiler that chooses to put the new_vmbuf[] local variable at an odd starting address, and a checksum-enabled database, pg_upgrade would dump core. Point one was reported by Thomas Kellerer, the other two detected by code-reading. Point two is much the nastiest of these issues from an impact standpoint, though fortunately it affects only a minority of users. The Windows issue will definitely bite people, but it seems quite unlikely that there would be undetected corruption from that. In addition, I failed to resist the temptation to do some minor cosmetic adjustments, mostly improving the comments. It would be a good idea to try to improve the error reporting here, but that seems like material for a separate patch. Discussion: --- src/bin/pg_upgrade/file.c | 91 +++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/src/bin/pg_upgrade/file.c b/src/bin/pg_upgrade/file.c index b33f0b46e3..3e04c1a33e 100644 --- a/src/bin/pg_upgrade/file.c +++ b/src/bin/pg_upgrade/file.c @@ -18,8 +18,6 @@ #include #include -#define BITS_PER_HEAPBLOCK_OLD 1 - #ifndef WIN32 static int copy_file(const char *fromfile, const char *tofile); @@ -84,10 +82,11 @@ copy_file(const char *srcfile, const char *dstfile) return -1; } - if ((src_fd = open(srcfile, O_RDONLY, 0)) < 0) + if ((src_fd = open(srcfile, O_RDONLY | PG_BINARY, 0)) < 0) return -1; - if ((dest_fd = open(dstfile, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR)) < 0) + if ((dest_fd = open(dstfile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, + S_IRUSR | S_IWUSR)) < 0) { save_errno = errno; @@ -153,31 +152,30 @@ copy_file(const char *srcfile, const char *dstfile) * version, we could refuse to copy visibility maps from the old cluster * to the new cluster; the next VACUUM would recreate them, but at the * price of scanning the entire table. So, instead, we rewrite the old - * visibility maps in the new format. That way, the all-visible bit - * remains set for the pages for which it was set previously. The - * all-frozen bit is never set by this conversion; we leave that to - * VACUUM. + * visibility maps in the new format. That way, the all-visible bits + * remain set for the pages for which they were set previously. The + * all-frozen bits are never set by this conversion; we leave that to VACUUM. */ const char * rewriteVisibilityMap(const char *fromfile, const char *tofile) { - int src_fd = 0; - int dst_fd = 0; - char buffer[BLCKSZ]; - ssize_t bytesRead; + int src_fd; + int dst_fd; + char *buffer; + char *new_vmbuf; ssize_t totalBytesRead = 0; ssize_t src_filesize; int rewriteVmBytesPerPage; BlockNumber new_blkno = 0; struct stat statbuf; - /* Compute we need how many old page bytes to rewrite a new page */ + /* Compute number of old-format bytes per new page */ rewriteVmBytesPerPage = (BLCKSZ - SizeOfPageHeaderData) / 2; if ((fromfile == NULL) || (tofile == NULL)) return "Invalid old file or new file"; - if ((src_fd = open(fromfile, O_RDONLY, 0)) < 0) + if ((src_fd = open(fromfile, O_RDONLY | PG_BINARY, 0)) < 0) return getErrorText(); if (fstat(src_fd, &statbuf) != 0) @@ -186,7 +184,8 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile) return getErrorText(); } - if ((dst_fd = open(tofile, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR)) < 0) + if ((dst_fd = open(tofile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, + S_IRUSR | S_IWUSR)) < 0) { close(src_fd); return getErrorText(); @@ -195,14 +194,22 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile) /* Save old file size */ src_filesize = statbuf.st_size; + /* + * Malloc the work buffers, rather than making them local arrays, to + * ensure adequate alignment. + */ + buffer = (char *) pg_malloc(BLCKSZ); + new_vmbuf = (char *) pg_malloc(BLCKSZ); + /* * Turn each visibility map page into 2 pages one by one. Each new page - * has the same page header as the old one. If the last section of last - * page is empty, we skip it, mostly to avoid turning one-page visibility - * maps for small relations into two pages needlessly. + * has the same page header as the old one. If the last section of the + * last page is empty, we skip it, mostly to avoid turning one-page + * visibility maps for small relations into two pages needlessly. */ while (totalBytesRead < src_filesize) { + ssize_t bytesRead; char *old_cur; char *old_break; char *old_blkend; @@ -225,61 +232,59 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile) /* * These old_* variables point to old visibility map page. old_cur * points to current position on old page. old_blkend points to end of - * old block. old_break points to old page break position for - * rewriting a new page. After wrote a new page, old_break proceeds - * rewriteVmBytesPerPage bytes. + * old block. old_break is the end+1 position on the old page for the + * data that will be transferred to the current new page. */ old_cur = buffer + SizeOfPageHeaderData; old_blkend = buffer + bytesRead; old_break = old_cur + rewriteVmBytesPerPage; - while (old_blkend >= old_break) + while (old_break <= old_blkend) { - char new_vmbuf[BLCKSZ]; - char *new_cur = new_vmbuf; + char *new_cur; bool empty = true; bool old_lastpart; - /* Copy page header in advance */ + /* First, copy old page header to new page */ memcpy(new_vmbuf, &pageheader, SizeOfPageHeaderData); - /* Rewrite the last part of the old page? */ - old_lastpart = old_lastblk && (old_blkend == old_break); + /* Rewriting the last part of the last old page? */ + old_lastpart = old_lastblk && (old_break == old_blkend); - new_cur += SizeOfPageHeaderData; + new_cur = new_vmbuf + SizeOfPageHeaderData; /* Process old page bytes one by one, and turn it into new page. */ - while (old_break > old_cur) + while (old_cur < old_break) { + uint8 byte = *(uint8 *) old_cur; uint16 new_vmbits = 0; int i; /* Generate new format bits while keeping old information */ for (i = 0; i < BITS_PER_BYTE; i++) { - uint8 byte = *(uint8 *) old_cur; - - if (byte & (1 << (BITS_PER_HEAPBLOCK_OLD * i))) + if (byte & (1 << i)) { empty = false; - new_vmbits |= 1 << (BITS_PER_HEAPBLOCK * i); + new_vmbits |= + VISIBILITYMAP_ALL_VISIBLE << (BITS_PER_HEAPBLOCK * i); } } - /* Copy new visibility map bit to new format page */ - memcpy(new_cur, &new_vmbits, BITS_PER_HEAPBLOCK); + /* Copy new visibility map bytes to new-format page */ + new_cur[0] = (char) (new_vmbits & 0xFF); + new_cur[1] = (char) (new_vmbits >> 8); - old_cur += BITS_PER_HEAPBLOCK_OLD; + old_cur++; new_cur += BITS_PER_HEAPBLOCK; } - /* If the last part of the old page is empty, skip writing it */ + /* If the last part of the last page is empty, skip writing it */ if (old_lastpart && empty) break; - /* Set new checksum for a visibility map page (if enabled) */ - if (old_cluster.controldata.data_checksum_version != 0 && - new_cluster.controldata.data_checksum_version != 0) + /* Set new checksum for visibility map page, if enabled */ + if (new_cluster.controldata.data_checksum_version != 0) ((PageHeader) new_vmbuf)->pd_checksum = pg_checksum_page(new_vmbuf, new_blkno); @@ -290,17 +295,19 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile) return getErrorText(); } + /* Advance for next new page */ old_break += rewriteVmBytesPerPage; new_blkno++; } } - /* Close files */ + /* Clean up */ + pg_free(buffer); + pg_free(new_vmbuf); close(dst_fd); close(src_fd); return NULL; - } void From f002ed2b8e45286fe73a36e119cba2016ea0de19 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 30 Sep 2016 20:40:27 -0400 Subject: [PATCH 253/871] Improve error reporting in pg_upgrade's file copying/linking/rewriting. The previous design for this had copyFile(), linkFile(), and rewriteVisibilityMap() returning strerror strings, with the caller producing one-size-fits-all error messages based on that. This made it impossible to produce messages that described the failures with any degree of precision, especially not short-read problems since those don't set errno at all. Since pg_upgrade has no intention of continuing after any error in this area, let's fix this by just letting these functions call pg_fatal() for themselves, making it easy for each point of failure to have a suitable error message. Taking this approach also allows dropping cleanup code that was unnecessary and was often rather sloppy about preserving errno. To not lose relevant info that was reported before, pass in the schema name and table name of the current table so that they can be included in the error reports. An additional problem was the use of getErrorText(), which was flat out wrong for all but a couple of call sites, because it unconditionally did "_dosmaperr(GetLastError())" on Windows. That's only appropriate when reporting an error from a Windows-native API, which only a couple of the callers were actually doing. Thus, even the reported strerror string would be unrelated to the actual failure in many cases on Windows. To fix, get rid of getErrorText() altogether, and just have call sites do strerror(errno) instead, since that's the way all the rest of our frontend programs do it. Add back the _dosmaperr() calls in the two places where that's actually appropriate. In passing, make assorted messages hew more closely to project style guidelines, notably by removing initial capitals in not-complete-sentence primary error messages. (I didn't make any effort to clean up places I didn't have another reason to touch, though.) Per discussion of a report from Thomas Kellerer. Back-patch to 9.6, but no further; given the relative infrequency of reports of problems here, it's not clear it's worth adapting the patch to older branches. Patch by me, but with credit to Alvaro Herrera for spotting the issue with getErrorText's misuse of _dosmaperr(). Discussion: --- src/bin/pg_upgrade/check.c | 32 +++--- src/bin/pg_upgrade/controldata.c | 4 +- src/bin/pg_upgrade/exec.c | 8 +- src/bin/pg_upgrade/file.c | 182 +++++++++++++------------------ src/bin/pg_upgrade/function.c | 6 +- src/bin/pg_upgrade/option.c | 4 +- src/bin/pg_upgrade/pg_upgrade.c | 2 +- src/bin/pg_upgrade/pg_upgrade.h | 11 +- src/bin/pg_upgrade/relfilenode.c | 42 +++---- src/bin/pg_upgrade/tablespace.c | 4 +- src/bin/pg_upgrade/util.c | 15 --- src/bin/pg_upgrade/version.c | 6 +- 12 files changed, 132 insertions(+), 184 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index ed41dee6a5..42bf4992a0 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -431,8 +431,8 @@ create_script_for_cluster_analyze(char **analyze_script_file_name) SCRIPT_PREFIX, SCRIPT_EXT); if ((script = fopen_priv(*analyze_script_file_name, "w")) == NULL) - pg_fatal("Could not open file \"%s\": %s\n", - *analyze_script_file_name, getErrorText()); + pg_fatal("could not open file \"%s\": %s\n", + *analyze_script_file_name, strerror(errno)); #ifndef WIN32 /* add shebang header */ @@ -486,8 +486,8 @@ create_script_for_cluster_analyze(char **analyze_script_file_name) #ifndef WIN32 if (chmod(*analyze_script_file_name, S_IRWXU) != 0) - pg_fatal("Could not add execute permission to file \"%s\": %s\n", - *analyze_script_file_name, getErrorText()); + pg_fatal("could not add execute permission to file \"%s\": %s\n", + *analyze_script_file_name, strerror(errno)); #endif termPQExpBuffer(&user_specification); @@ -559,8 +559,8 @@ create_script_for_old_cluster_deletion(char **deletion_script_file_name) prep_status("Creating script to delete old cluster"); if ((script = fopen_priv(*deletion_script_file_name, "w")) == NULL) - pg_fatal("Could not open file \"%s\": %s\n", - *deletion_script_file_name, getErrorText()); + pg_fatal("could not open file \"%s\": %s\n", + *deletion_script_file_name, strerror(errno)); #ifndef WIN32 /* add shebang header */ @@ -615,8 +615,8 @@ create_script_for_old_cluster_deletion(char **deletion_script_file_name) #ifndef WIN32 if (chmod(*deletion_script_file_name, S_IRWXU) != 0) - pg_fatal("Could not add execute permission to file \"%s\": %s\n", - *deletion_script_file_name, getErrorText()); + pg_fatal("could not add execute permission to file \"%s\": %s\n", + *deletion_script_file_name, strerror(errno)); #endif check_ok(); @@ -819,8 +819,8 @@ check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster) { found = true; if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("Could not open file \"%s\": %s\n", - output_path, getErrorText()); + pg_fatal("could not open file \"%s\": %s\n", + output_path, strerror(errno)); if (!db_used) { fprintf(script, "Database: %s\n", active_db->db_name); @@ -922,8 +922,8 @@ check_for_reg_data_type_usage(ClusterInfo *cluster) { found = true; if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("Could not open file \"%s\": %s\n", - output_path, getErrorText()); + pg_fatal("could not open file \"%s\": %s\n", + output_path, strerror(errno)); if (!db_used) { fprintf(script, "Database: %s\n", active_db->db_name); @@ -1013,8 +1013,8 @@ check_for_jsonb_9_4_usage(ClusterInfo *cluster) { found = true; if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("Could not open file \"%s\": %s\n", - output_path, getErrorText()); + pg_fatal("could not open file \"%s\": %s\n", + output_path, strerror(errno)); if (!db_used) { fprintf(script, "Database: %s\n", active_db->db_name); @@ -1089,8 +1089,8 @@ get_bin_version(ClusterInfo *cluster) if ((output = popen(cmd, "r")) == NULL || fgets(cmd_output, sizeof(cmd_output), output) == NULL) - pg_fatal("Could not get pg_ctl version data using %s: %s\n", - cmd, getErrorText()); + pg_fatal("could not get pg_ctl version data using %s: %s\n", + cmd, strerror(errno)); pclose(output); diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c index d89cf196ab..709f149a9e 100644 --- a/src/bin/pg_upgrade/controldata.c +++ b/src/bin/pg_upgrade/controldata.c @@ -119,8 +119,8 @@ get_control_data(ClusterInfo *cluster, bool live_check) fflush(stderr); if ((output = popen(cmd, "r")) == NULL) - pg_fatal("Could not get control data using %s: %s\n", - cmd, getErrorText()); + pg_fatal("could not get control data using %s: %s\n", + cmd, strerror(errno)); /* Only in <= 9.2 */ if (GET_MAJOR_VERSION(cluster->major_version) <= 902) diff --git a/src/bin/pg_upgrade/exec.c b/src/bin/pg_upgrade/exec.c index dd30952441..6d04e5671d 100644 --- a/src/bin/pg_upgrade/exec.c +++ b/src/bin/pg_upgrade/exec.c @@ -191,7 +191,7 @@ pid_lock_file_exists(const char *datadir) /* ENOTDIR means we will throw a more useful error later */ if (errno != ENOENT && errno != ENOTDIR) pg_fatal("could not open file \"%s\" for reading: %s\n", - path, getErrorText()); + path, strerror(errno)); return false; } @@ -285,7 +285,7 @@ check_data_dir(const char *pg_data) if (stat(subDirName, &statBuf) != 0) report_status(PG_FATAL, "check for \"%s\" failed: %s\n", - subDirName, getErrorText()); + subDirName, strerror(errno)); else if (!S_ISDIR(statBuf.st_mode)) report_status(PG_FATAL, "%s is not a directory\n", subDirName); @@ -309,7 +309,7 @@ check_bin_dir(ClusterInfo *cluster) /* check bindir */ if (stat(cluster->bindir, &statBuf) != 0) report_status(PG_FATAL, "check for \"%s\" failed: %s\n", - cluster->bindir, getErrorText()); + cluster->bindir, strerror(errno)); else if (!S_ISDIR(statBuf.st_mode)) report_status(PG_FATAL, "%s is not a directory\n", cluster->bindir); @@ -352,7 +352,7 @@ validate_exec(const char *dir, const char *cmdName) */ if (stat(path, &buf) < 0) pg_fatal("check for \"%s\" failed: %s\n", - path, getErrorText()); + path, strerror(errno)); else if (!S_ISREG(buf.st_mode)) pg_fatal("check for \"%s\" failed: not an executable file\n", path); diff --git a/src/bin/pg_upgrade/file.c b/src/bin/pg_upgrade/file.c index 3e04c1a33e..3461927992 100644 --- a/src/bin/pg_upgrade/file.c +++ b/src/bin/pg_upgrade/file.c @@ -19,9 +19,7 @@ #include -#ifndef WIN32 -static int copy_file(const char *fromfile, const char *tofile); -#else +#ifdef WIN32 static int win32_pghardlink(const char *src, const char *dst); #endif @@ -29,73 +27,29 @@ static int win32_pghardlink(const char *src, const char *dst); /* * copyFile() * - * Copies a relation file from src to dst. + * Copies a relation file from src to dst. + * schemaName/relName are relation's SQL name (used for error messages only). */ -const char * -copyFile(const char *src, const char *dst) +void +copyFile(const char *src, const char *dst, + const char *schemaName, const char *relName) { #ifndef WIN32 - if (copy_file(src, dst) == -1) -#else - if (CopyFile(src, dst, true) == 0) -#endif - return getErrorText(); - else - return NULL; -} - - -/* - * linkFile() - * - * Creates a hard link between the given relation files. We use - * this function to perform a true in-place update. If the on-disk - * format of the new cluster is bit-for-bit compatible with the on-disk - * format of the old cluster, we can simply link each relation - * instead of copying the data from the old cluster to the new cluster. - */ -const char * -linkFile(const char *src, const char *dst) -{ - if (pg_link_file(src, dst) == -1) - return getErrorText(); - else - return NULL; -} - - -#ifndef WIN32 -static int -copy_file(const char *srcfile, const char *dstfile) -{ -#define COPY_BUF_SIZE (50 * BLCKSZ) - int src_fd; int dest_fd; char *buffer; - int ret = 0; - int save_errno = 0; - - if ((srcfile == NULL) || (dstfile == NULL)) - { - errno = EINVAL; - return -1; - } - if ((src_fd = open(srcfile, O_RDONLY | PG_BINARY, 0)) < 0) - return -1; + if ((src_fd = open(src, O_RDONLY | PG_BINARY, 0)) < 0) + pg_fatal("error while copying relation \"%s.%s\": could not open file \"%s\": %s\n", + schemaName, relName, src, strerror(errno)); - if ((dest_fd = open(dstfile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, + if ((dest_fd = open(dst, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, S_IRUSR | S_IWUSR)) < 0) - { - save_errno = errno; + pg_fatal("error while copying relation \"%s.%s\": could not create file \"%s\": %s\n", + schemaName, relName, dst, strerror(errno)); - if (src_fd != 0) - close(src_fd); - - errno = save_errno; - return -1; - } + /* copy in fairly large chunks for best efficiency */ +#define COPY_BUF_SIZE (50 * BLCKSZ) buffer = (char *) pg_malloc(COPY_BUF_SIZE); @@ -105,47 +59,62 @@ copy_file(const char *srcfile, const char *dstfile) ssize_t nbytes = read(src_fd, buffer, COPY_BUF_SIZE); if (nbytes < 0) - { - save_errno = errno; - ret = -1; - break; - } + pg_fatal("error while copying relation \"%s.%s\": could not read file \"%s\": %s\n", + schemaName, relName, src, strerror(errno)); if (nbytes == 0) break; errno = 0; - if (write(dest_fd, buffer, nbytes) != nbytes) { /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) errno = ENOSPC; - save_errno = errno; - ret = -1; - break; + pg_fatal("error while copying relation \"%s.%s\": could not write file \"%s\": %s\n", + schemaName, relName, dst, strerror(errno)); } } pg_free(buffer); + close(src_fd); + close(dest_fd); - if (src_fd != 0) - close(src_fd); +#else /* WIN32 */ + + if (CopyFile(src, dst, true) == 0) + { + _dosmaperr(GetLastError()); + pg_fatal("error while copying relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n", + schemaName, relName, src, dst, strerror(errno)); + } - if (dest_fd != 0) - close(dest_fd); +#endif /* WIN32 */ +} - if (save_errno != 0) - errno = save_errno; - return ret; +/* + * linkFile() + * + * Hard-links a relation file from src to dst. + * schemaName/relName are relation's SQL name (used for error messages only). + */ +void +linkFile(const char *src, const char *dst, + const char *schemaName, const char *relName) +{ + if (pg_link_file(src, dst) < 0) + pg_fatal("error while creating link for relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n", + schemaName, relName, src, dst, strerror(errno)); } -#endif /* * rewriteVisibilityMap() * + * Transform a visibility map file, copying from src to dst. + * schemaName/relName are relation's SQL name (used for error messages only). + * * In versions of PostgreSQL prior to catversion 201603011, PostgreSQL's * visibility map included one bit per heap page; it now includes two. * When upgrading a cluster from before that time to a current PostgreSQL @@ -156,8 +125,9 @@ copy_file(const char *srcfile, const char *dstfile) * remain set for the pages for which they were set previously. The * all-frozen bits are never set by this conversion; we leave that to VACUUM. */ -const char * -rewriteVisibilityMap(const char *fromfile, const char *tofile) +void +rewriteVisibilityMap(const char *fromfile, const char *tofile, + const char *schemaName, const char *relName) { int src_fd; int dst_fd; @@ -172,24 +142,18 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile) /* Compute number of old-format bytes per new page */ rewriteVmBytesPerPage = (BLCKSZ - SizeOfPageHeaderData) / 2; - if ((fromfile == NULL) || (tofile == NULL)) - return "Invalid old file or new file"; - if ((src_fd = open(fromfile, O_RDONLY | PG_BINARY, 0)) < 0) - return getErrorText(); + pg_fatal("error while copying relation \"%s.%s\": could not open file \"%s\": %s\n", + schemaName, relName, fromfile, strerror(errno)); if (fstat(src_fd, &statbuf) != 0) - { - close(src_fd); - return getErrorText(); - } + pg_fatal("error while copying relation \"%s.%s\": could not stat file \"%s\": %s\n", + schemaName, relName, fromfile, strerror(errno)); if ((dst_fd = open(tofile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, S_IRUSR | S_IWUSR)) < 0) - { - close(src_fd); - return getErrorText(); - } + pg_fatal("error while copying relation \"%s.%s\": could not create file \"%s\": %s\n", + schemaName, relName, tofile, strerror(errno)); /* Save old file size */ src_filesize = statbuf.st_size; @@ -218,9 +182,12 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile) if ((bytesRead = read(src_fd, buffer, BLCKSZ)) != BLCKSZ) { - close(dst_fd); - close(src_fd); - return getErrorText(); + if (bytesRead < 0) + pg_fatal("error while copying relation \"%s.%s\": could not read file \"%s\": %s\n", + schemaName, relName, fromfile, strerror(errno)); + else + pg_fatal("error while copying relation \"%s.%s\": partial page found in file \"%s\"\n", + schemaName, relName, fromfile); } totalBytesRead += BLCKSZ; @@ -288,11 +255,14 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile) ((PageHeader) new_vmbuf)->pd_checksum = pg_checksum_page(new_vmbuf, new_blkno); + errno = 0; if (write(dst_fd, new_vmbuf, BLCKSZ) != BLCKSZ) { - close(dst_fd); - close(src_fd); - return getErrorText(); + /* if write didn't set errno, assume problem is no disk space */ + if (errno == 0) + errno = ENOSPC; + pg_fatal("error while copying relation \"%s.%s\": could not write file \"%s\": %s\n", + schemaName, relName, tofile, strerror(errno)); } /* Advance for next new page */ @@ -306,8 +276,6 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile) pg_free(new_vmbuf); close(dst_fd); close(src_fd); - - return NULL; } void @@ -320,16 +288,16 @@ check_hard_link(void) snprintf(new_link_file, sizeof(new_link_file), "%s/PG_VERSION.linktest", new_cluster.pgdata); unlink(new_link_file); /* might fail */ - if (pg_link_file(existing_file, new_link_file) == -1) - { - pg_fatal("Could not create hard link between old and new data directories: %s\n" + if (pg_link_file(existing_file, new_link_file) < 0) + pg_fatal("could not create hard link between old and new data directories: %s\n" "In link mode the old and new data directories must be on the same file system volume.\n", - getErrorText()); - } + strerror(errno)); + unlink(new_link_file); } #ifdef WIN32 +/* implementation of pg_link_file() on Windows */ static int win32_pghardlink(const char *src, const char *dst) { @@ -338,7 +306,10 @@ win32_pghardlink(const char *src, const char *dst) * http://msdn.microsoft.com/en-us/library/aa363860(VS.85).aspx */ if (CreateHardLinkA(dst, src, NULL) == 0) + { + _dosmaperr(GetLastError()); return -1; + } else return 0; } @@ -353,7 +324,8 @@ fopen_priv(const char *path, const char *mode) FILE *fp; fp = fopen(path, mode); - umask(old_umask); + + umask(old_umask); /* we assume this can't change errno */ return fp; } diff --git a/src/bin/pg_upgrade/function.c b/src/bin/pg_upgrade/function.c index eaae976c53..b432b544b2 100644 --- a/src/bin/pg_upgrade/function.c +++ b/src/bin/pg_upgrade/function.c @@ -213,9 +213,9 @@ check_loadable_libraries(void) found = true; if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("Could not open file \"%s\": %s\n", - output_path, getErrorText()); - fprintf(script, "Could not load library \"%s\"\n%s\n", + pg_fatal("could not open file \"%s\": %s\n", + output_path, strerror(errno)); + fprintf(script, "could not load library \"%s\":\n%s\n", lib, PQerrorMessage(conn)); } diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c index 4597355461..2e9a40c2b6 100644 --- a/src/bin/pg_upgrade/option.c +++ b/src/bin/pg_upgrade/option.c @@ -422,8 +422,8 @@ adjust_data_dir(ClusterInfo *cluster) if ((output = popen(cmd, "r")) == NULL || fgets(cmd_output, sizeof(cmd_output), output) == NULL) - pg_fatal("Could not get data directory using %s: %s\n", - cmd, getErrorText()); + pg_fatal("could not get data directory using %s: %s\n", + cmd, strerror(errno)); pclose(output); diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index fa118e9e4f..90c07205bf 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -225,7 +225,7 @@ setup(char *argv0, bool *live_check) /* get path to pg_upgrade executable */ if (find_my_exec(argv0, exec_path) < 0) - pg_fatal("Could not get path name to pg_upgrade: %s\n", getErrorText()); + pg_fatal("%s: could not find own program executable\n", argv0); /* Trim off program name and keep just path */ *last_dir_separator(exec_path) = '\0'; diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 20f9a91087..19dca83386 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -367,10 +367,12 @@ bool pid_lock_file_exists(const char *datadir); /* file.c */ -const char *copyFile(const char *src, const char *dst); -const char *linkFile(const char *src, const char *dst); -const char *rewriteVisibilityMap(const char *fromfile, const char *tofile); - +void copyFile(const char *src, const char *dst, + const char *schemaName, const char *relName); +void linkFile(const char *src, const char *dst, + const char *schemaName, const char *relName); +void rewriteVisibilityMap(const char *fromfile, const char *tofile, + const char *schemaName, const char *relName); void check_hard_link(void); FILE *fopen_priv(const char *path, const char *mode); @@ -431,7 +433,6 @@ void pg_fatal(const char *fmt,...) pg_attribute_printf(1, 2) pg_attribute_noret void end_progress_output(void); void prep_status(const char *fmt,...) pg_attribute_printf(1, 2); void check_ok(void); -const char *getErrorText(void); unsigned int str2uint(const char *str); void pg_putenv(const char *var, const char *val); diff --git a/src/bin/pg_upgrade/relfilenode.c b/src/bin/pg_upgrade/relfilenode.c index 85cb717f74..c8c2a28f4e 100644 --- a/src/bin/pg_upgrade/relfilenode.c +++ b/src/bin/pg_upgrade/relfilenode.c @@ -183,7 +183,6 @@ transfer_single_new_db(FileNameMap *maps, int size, char *old_tablespace) static void transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_frozenbit) { - const char *msg; char old_file[MAXPGPATH]; char new_file[MAXPGPATH]; int segno; @@ -229,7 +228,7 @@ transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_fro else pg_fatal("error while checking for file existence \"%s.%s\" (\"%s\" to \"%s\"): %s\n", map->nspname, map->relname, old_file, new_file, - getErrorText()); + strerror(errno)); } /* If file is empty, just return */ @@ -242,35 +241,24 @@ transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_fro /* Copying files might take some time, so give feedback. */ pg_log(PG_STATUS, "%s", old_file); - if (user_opts.transfer_mode == TRANSFER_MODE_COPY) + if (vm_must_add_frozenbit && strcmp(type_suffix, "_vm") == 0) { - pg_log(PG_VERBOSE, "copying \"%s\" to \"%s\"\n", old_file, new_file); - - /* Rewrite visibility map if needed */ - if (vm_must_add_frozenbit && (strcmp(type_suffix, "_vm") == 0)) - msg = rewriteVisibilityMap(old_file, new_file); - else - msg = copyFile(old_file, new_file); - - if (msg) - pg_fatal("error while copying relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n", - map->nspname, map->relname, old_file, new_file, msg); + /* Need to rewrite visibility map format */ + pg_log(PG_VERBOSE, "rewriting \"%s\" to \"%s\"\n", + old_file, new_file); + rewriteVisibilityMap(old_file, new_file, map->nspname, map->relname); + } + else if (user_opts.transfer_mode == TRANSFER_MODE_COPY) + { + pg_log(PG_VERBOSE, "copying \"%s\" to \"%s\"\n", + old_file, new_file); + copyFile(old_file, new_file, map->nspname, map->relname); } else { - pg_log(PG_VERBOSE, "linking \"%s\" to \"%s\"\n", old_file, new_file); - - /* Rewrite visibility map if needed */ - if (vm_must_add_frozenbit && (strcmp(type_suffix, "_vm") == 0)) - msg = rewriteVisibilityMap(old_file, new_file); - else - msg = linkFile(old_file, new_file); - - if (msg) - pg_fatal("error while creating link for relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n", - map->nspname, map->relname, old_file, new_file, msg); + pg_log(PG_VERBOSE, "linking \"%s\" to \"%s\"\n", + old_file, new_file); + linkFile(old_file, new_file, map->nspname, map->relname); } } - - return; } diff --git a/src/bin/pg_upgrade/tablespace.c b/src/bin/pg_upgrade/tablespace.c index dfbce59ca3..8e09ec3729 100644 --- a/src/bin/pg_upgrade/tablespace.c +++ b/src/bin/pg_upgrade/tablespace.c @@ -90,8 +90,8 @@ get_tablespace_paths(void) os_info.old_tablespaces[tblnum]); else report_status(PG_FATAL, - "cannot stat() tablespace directory \"%s\": %s\n", - os_info.old_tablespaces[tblnum], getErrorText()); + "could not stat tablespace directory \"%s\": %s\n", + os_info.old_tablespaces[tblnum], strerror(errno)); } if (!S_ISDIR(statBuf.st_mode)) report_status(PG_FATAL, diff --git a/src/bin/pg_upgrade/util.c b/src/bin/pg_upgrade/util.c index 5b24ec0917..aadc1cdd9d 100644 --- a/src/bin/pg_upgrade/util.c +++ b/src/bin/pg_upgrade/util.c @@ -232,21 +232,6 @@ get_user_info(char **user_name_p) } -/* - * getErrorText() - * - * Returns the text of the most recent error - */ -const char * -getErrorText(void) -{ -#ifdef WIN32 - _dosmaperr(GetLastError()); -#endif - return pg_strdup(strerror(errno)); -} - - /* * str2uint() * diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 36c299c101..3c7c5facf4 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -52,7 +52,8 @@ new_9_0_populate_pg_largeobject_metadata(ClusterInfo *cluster, bool check_mode) PQExpBufferData connectbuf; if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s\n", output_path, getErrorText()); + pg_fatal("could not open file \"%s\": %s\n", output_path, + strerror(errno)); initPQExpBuffer(&connectbuf); appendPsqlMetaConnect(&connectbuf, active_db->db_name); @@ -150,7 +151,8 @@ old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) { found = true; if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s\n", output_path, getErrorText()); + pg_fatal("could not open file \"%s\": %s\n", output_path, + strerror(errno)); if (!db_used) { fprintf(script, "Database: %s\n", active_db->db_name); From a4327296df7366ecc657b706a9b5e87aa921311a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 30 Sep 2016 12:00:00 -0400 Subject: [PATCH 254/871] Set log_line_prefix and application name in test drivers Before pg_regress runs psql, set the application name to the test name. Similarly, set the application name to the test file name in the TAP tests. Also, set a default log_line_prefix that show the application name, as well as the PID and a time stamp. That way, the server log output can be correlated to the test input files, making debugging a bit easier. --- src/test/perl/PostgresNode.pm | 1 + src/test/perl/TestLib.pm | 2 ++ src/test/regress/pg_regress.c | 1 + src/test/regress/pg_regress_main.c | 7 +++++++ 4 files changed, 11 insertions(+) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 1611ac9461..4b225e1471 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -409,6 +409,7 @@ sub init open my $conf, ">>$pgdata/postgresql.conf"; print $conf "\n# Added by PostgresNode.pm\n"; print $conf "fsync = off\n"; + print $conf "log_line_prefix = '%m [%p] %q%a '\n"; print $conf "log_statement = all\n"; print $conf "port = $port\n"; diff --git a/src/test/perl/TestLib.pm b/src/test/perl/TestLib.pm index 31e7acd4da..d22957ceb0 100644 --- a/src/test/perl/TestLib.pm +++ b/src/test/perl/TestLib.pm @@ -62,6 +62,8 @@ BEGIN delete $ENV{PGPORT}; delete $ENV{PGHOST}; + $ENV{PGAPPNAME} = $0; + # Must be set early $windows_os = $Config{osname} eq 'MSWin32' || $Config{osname} eq 'msys'; } diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index b28cb0b1e1..762adb8443 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -2270,6 +2270,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc fputs("\n# Configuration added by pg_regress\n\n", pg_conf); fputs("log_autovacuum_min_duration = 0\n", pg_conf); fputs("log_checkpoints = on\n", pg_conf); + fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf); fputs("log_lock_waits = on\n", pg_conf); fputs("log_temp_files = 128kB\n", pg_conf); fputs("max_prepared_transactions = 2\n", pg_conf); diff --git a/src/test/regress/pg_regress_main.c b/src/test/regress/pg_regress_main.c index d9591c0608..273363588a 100644 --- a/src/test/regress/pg_regress_main.c +++ b/src/test/regress/pg_regress_main.c @@ -34,6 +34,7 @@ psql_start_test(const char *testname, char expectfile[MAXPGPATH]; char psql_cmd[MAXPGPATH * 3]; size_t offset = 0; + char *appnameenv; /* * Look for files in the output dir first, consistent with a vpath search. @@ -63,6 +64,9 @@ psql_start_test(const char *testname, offset += snprintf(psql_cmd + offset, sizeof(psql_cmd) - offset, "%s ", launcher); + appnameenv = psprintf("PGAPPNAME=pg_regress/%s", testname); + putenv(appnameenv); + snprintf(psql_cmd + offset, sizeof(psql_cmd) - offset, "\"%s%spsql\" -X -a -q -d \"%s\" < \"%s\" > \"%s\" 2>&1", bindir ? bindir : "", @@ -80,6 +84,9 @@ psql_start_test(const char *testname, exit(2); } + unsetenv("PGAPPNAME"); + free(appnameenv); + return pid; } From 7107d58ec5a3c45967e77525809612a5f89b97f3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 1 Oct 2016 13:35:13 -0400 Subject: [PATCH 255/871] Fix misplacement of submake-generated-headers prerequisites. The sequence "configure; cd src/pl/plpython; make -j" failed due to trying to compile plpython's .o files before the generated headers finished building. (This is an important real-world case, since it's the typical second step when building both plpython2 and plpython3.) This happens because the submake-generated-headers target is not placed in a way to make it a prerequisite to compiling the .o files. Fix that. Checking other uses of submake-generated-headers, I noted that the one attached to pg_regress was similarly misplaced; but it's actually not needed at all for pg_regress.o, rather regress.o, so move it to be a prerequisite of that. Back-patch to 9.6 where submake-generated-headers was introduced (by commit 548af97fc). It's not immediately clear to me why the previous coding didn't have the same issue; but since we've not had field reports of plpython make failing, leave it alone in the older branches. Pavel Raiskup and Tom Lane Discussion: <1925924.izSMJEZO3x@unused-4-107.brq.redhat.com> --- src/pl/plpython/Makefile | 4 +++- src/test/regress/GNUmakefile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pl/plpython/Makefile b/src/pl/plpython/Makefile index 647b4b1b96..7680d49cb6 100644 --- a/src/pl/plpython/Makefile +++ b/src/pl/plpython/Makefile @@ -95,7 +95,9 @@ REGRESS_PLPYTHON3_MANGLE := $(REGRESS) include $(top_srcdir)/src/Makefile.shlib -all: submake-generated-headers all-lib +all: all-lib + +$(OBJS): | submake-generated-headers install: all install-lib install-data diff --git a/src/test/regress/GNUmakefile b/src/test/regress/GNUmakefile index 6a275cb729..469b0937a2 100644 --- a/src/test/regress/GNUmakefile +++ b/src/test/regress/GNUmakefile @@ -36,7 +36,7 @@ EXTRADEFS = '-DHOST_TUPLE="$(host_tuple)"' \ all: pg_regress$(X) -pg_regress$(X): pg_regress.o pg_regress_main.o $(WIN32RES) | submake-libpgport submake-generated-headers +pg_regress$(X): pg_regress.o pg_regress_main.o $(WIN32RES) | submake-libpgport $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@ # dependencies ensure that path changes propagate @@ -65,6 +65,8 @@ include $(top_srcdir)/src/Makefile.shlib all: all-lib +$(OBJS): | submake-generated-headers + # Test input and expected files. These are created by pg_regress itself, so we # don't have a rule to create them. We do need rules to clean them however. input_files = $(patsubst $(srcdir)/input/%.source,sql/%.sql, $(wildcard $(srcdir)/input/*.source)) From ea046f08d1bcee56c3bedfa16a05c38a0515b41d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 1 Oct 2016 13:45:16 -0400 Subject: [PATCH 256/871] Fix misstatement in comment in Makefile.shlib. There is no need for "all: all-lib" to be placed before inclusion of Makefile.shlib. Makefile.global is what ensures that "all" is the default target, and we already document that that has to be included first. Per comment from Pavel Raiskup. Discussion: <1925924.izSMJEZO3x@unused-4-107.brq.redhat.com> --- src/Makefile.shlib | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Makefile.shlib b/src/Makefile.shlib index de93f41639..f64eb4d9c5 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -47,9 +47,8 @@ # clean-lib delete the static and shared libraries from the build dir # maintainer-clean-lib delete .def files built for win32 # -# Since `all-lib' is the first rule in this file you probably want to -# have the `all' target before including this file. In the most simple -# case it would look like this: +# Typically you would add `all-lib' to the `all' target so that `make all' +# builds the libraries. In the most simple case it would look like this: # # all: all-lib # From 33596edf09516a7cab65914e16cfd6adf9fc55d1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 1 Oct 2016 15:32:53 -0400 Subject: [PATCH 257/871] Copy-editing for contrib/pg_visibility documentation. Add omitted names for some function parameters. Fix some minor grammatical issues. --- doc/src/sgml/pgvisibility.sgml | 87 +++++++++++++++++----------------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/doc/src/sgml/pgvisibility.sgml b/doc/src/sgml/pgvisibility.sgml index d764eff9a0..fd486696fc 100644 --- a/doc/src/sgml/pgvisibility.sgml +++ b/doc/src/sgml/pgvisibility.sgml @@ -9,31 +9,33 @@ The pg_visibility module provides a means for examining the - visibility map (VM) and page-level visibility information. It also - provides functions to check the integrity of the visibility map and to + visibility map (VM) and page-level visibility information of a table. + It also provides functions to check the integrity of a visibility map and to force it to be rebuilt. Three different bits are used to store information about page-level visibility. The all-visible bit in the visibility map indicates that every - tuple on a given page of a relation is visible to every current transaction. - The all-frozen bit in the visibility map indicates that every tuple on the - page is frozen; that is, no future vacuum will need to modify the page - until such time as a tuple is inserted, updated, deleted, or locked on - that page. The page-level PD_ALL_VISIBLE bit has the + tuple in the corresponding page of the relation is visible to every current + and future transaction. The all-frozen bit in the visibility map indicates + that every tuple in the page is frozen; that is, no future vacuum will need + to modify the page until such time as a tuple is inserted, updated, deleted, + or locked on that page. + The page header's PD_ALL_VISIBLE bit has the same meaning as the all-visible bit in the visibility map, but is stored - within the data page itself rather than a separate data structure. These - will normally agree, but the page-level bit can sometimes be set while the - visibility map bit is clear after a crash recovery; or they can disagree - because of a change which occurs after pg_visibility examines - the visibility map and before it examines the data page. Any event which - causes data corruption can also cause these bits to disagree. + within the data page itself rather than in a separate data structure. + These two bits will normally agree, but the page's all-visible bit can + sometimes be set while the visibility map bit is clear after a crash + recovery. The reported values can also disagree because of a change that + occurs after pg_visibility examines the visibility map and + before it examines the data page. Any event that causes data corruption + can also cause these bits to disagree. - Functions which display information about PD_ALL_VISIBLE - are much more costly than those which only consult the visibility map, + Functions that display information about PD_ALL_VISIBLE bits + are much more costly than those that only consult the visibility map, because they must read the relation's data blocks rather than only the (much smaller) visibility map. Functions that check the relation's data blocks are similarly expensive. @@ -44,7 +46,7 @@ - pg_visibility_map(regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean) returns record + pg_visibility_map(relation regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean) returns record Returns the all-visible and all-frozen bits in the visibility map for @@ -54,40 +56,40 @@ - pg_visibility(regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns record + pg_visibility(relation regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns record Returns the all-visible and all-frozen bits in the visibility map for the given block of the given relation, plus the - PD_ALL_VISIBLE bit for that block. + PD_ALL_VISIBLE bit of that block. - pg_visibility_map(regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean) returns setof record + pg_visibility_map(relation regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean) returns setof record Returns the all-visible and all-frozen bits in the visibility map for - each block the given relation. + each block of the given relation. - pg_visibility(regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns setof record + pg_visibility(relation regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns setof record Returns the all-visible and all-frozen bits in the visibility map for - each block the given relation, plus the PD_ALL_VISIBLE - bit for each block. + each block of the given relation, plus the PD_ALL_VISIBLE + bit of each block. - pg_visibility_map_summary(regclass, all_visible OUT bigint, all_frozen OUT bigint) returns record + pg_visibility_map_summary(relation regclass, all_visible OUT bigint, all_frozen OUT bigint) returns record @@ -96,50 +98,49 @@ - + - pg_check_frozen(regclass, t_ctid OUT tid) returns setof tid + pg_check_frozen(relation regclass, t_ctid OUT tid) returns setof tid - Returns the TIDs of non-frozen tuples present in pages marked all-frozen + Returns the TIDs of non-frozen tuples stored in pages marked all-frozen in the visibility map. If this function returns a non-empty set of - TIDs, the database is corrupt. + TIDs, the visibility map is corrupt. - - - pg_check_visible(regclass, t_ctid OUT tid) returns setof tid + + + pg_check_visible(relation regclass, t_ctid OUT tid) returns setof tid - Returns the TIDs of tuples which are not all-visible despite the fact - that the pages which contain them are marked as all-visible in the - visibility map. If this function returns a non-empty set of TIDs, the - database is corrupt. + Returns the TIDs of non-all-visible tuples stored in pages marked + all-visible in the visibility map. If this function returns a non-empty + set of TIDs, the visibility map is corrupt. - pg_truncate_visibility_map(regclass) returns void + pg_truncate_visibility_map(relation regclass) returns void - Truncates the visibility map for the given relation. This function - is only expected to be useful if you suspect that the visibility map - for the indicated relation is corrupt and wish to rebuild it. The first - VACUUM executed on the given relation after this function - is executed will scan every page in the relation and rebuild the - visibility map. + Truncates the visibility map for the given relation. This function is + useful if you believe that the visibility map for the relation is + corrupt and wish to force rebuilding it. The first VACUUM + executed on the given relation after this function is executed will scan + every page in the relation and rebuild the visibility map. (Until that + is done, queries will treat the visibility map as containing all zeroes.) - By default, these functions are not publicly executable. + By default, these functions are executable only by superusers. From 9a109452da1b923e183f20fcf5516984d448ece9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 1 Oct 2016 16:32:54 -0400 Subject: [PATCH 258/871] Fix bugs in contrib/pg_visibility. collect_corrupt_items() failed to initialize tuple.t_self. While HeapTupleSatisfiesVacuum() doesn't actually use that value, it does Assert that it's valid, so that the code would dump core if ip_posid chanced to be zero. (That's somewhat unlikely, which probably explains how this got missed. In any case it wouldn't matter for field use.) Also, collect_corrupt_items was returning the wrong TIDs, that is the contents of t_ctid rather than the tuple's own location. This would be the same thing in simple cases, but it could be wrong if, for example, a past update attempt had been rolled back, leaving a live tuple whose t_ctid doesn't point at itself. Also, in pg_visibility(), guard against trying to read a page past the end of the rel. The VM code handles inquiries beyond the end of the map by silently returning zeroes, and it seems like we should do the same thing here. I ran into the assertion failure while using pg_visibility to check pg_upgrade's behavior, and then noted the other problems while reading the code. Report: <29043.1475288648@sss.pgh.pa.us> --- contrib/pg_visibility/pg_visibility.c | 38 ++++++++++++++++++--------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c index 7034066663..9985e3e3a1 100644 --- a/contrib/pg_visibility/pg_visibility.c +++ b/contrib/pg_visibility/pg_visibility.c @@ -3,6 +3,8 @@ * pg_visibility.c * display visibility map information and page-level visibility bits * + * Copyright (c) 2016, PostgreSQL Global Development Group + * * contrib/pg_visibility/pg_visibility.c *------------------------------------------------------------------------- */ @@ -54,6 +56,10 @@ static bool tuple_all_visible(HeapTuple tup, TransactionId OldestXmin, /* * Visibility map information for a single block of a relation. + * + * Note: the VM code will silently return zeroes for pages past the end + * of the map, so we allow probes up to MaxBlockNumber regardless of the + * actual relation size. */ Datum pg_visibility_map(PG_FUNCTION_ARGS) @@ -122,13 +128,22 @@ pg_visibility(PG_FUNCTION_ARGS) values[0] = BoolGetDatum((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0); values[1] = BoolGetDatum((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0); - buffer = ReadBuffer(rel, blkno); - LockBuffer(buffer, BUFFER_LOCK_SHARE); + /* Here we have to explicitly check rel size ... */ + if (blkno < RelationGetNumberOfBlocks(rel)) + { + buffer = ReadBuffer(rel, blkno); + LockBuffer(buffer, BUFFER_LOCK_SHARE); - page = BufferGetPage(buffer); - values[2] = BoolGetDatum(PageIsAllVisible(page)); + page = BufferGetPage(buffer); + values[2] = BoolGetDatum(PageIsAllVisible(page)); - UnlockReleaseBuffer(buffer); + UnlockReleaseBuffer(buffer); + } + else + { + /* As with the vismap, silently return 0 for pages past EOF */ + values[2] = BoolGetDatum(false); + } relation_close(rel, AccessShareLock); @@ -611,14 +626,13 @@ collect_corrupt_items(Oid relid, bool all_visible, bool all_frozen) /* Dead line pointers are neither all-visible nor frozen. */ if (ItemIdIsDead(itemid)) { - ItemPointerData tid; - - ItemPointerSet(&tid, blkno, offnum); - record_corrupt_item(items, &tid); + ItemPointerSet(&(tuple.t_self), blkno, offnum); + record_corrupt_item(items, &tuple.t_self); continue; } /* Initialize a HeapTupleData structure for checks below. */ + ItemPointerSet(&(tuple.t_self), blkno, offnum); tuple.t_data = (HeapTupleHeader) PageGetItem(page, itemid); tuple.t_len = ItemIdGetLength(itemid); tuple.t_tableOid = relid; @@ -649,12 +663,12 @@ collect_corrupt_items(Oid relid, bool all_visible, bool all_frozen) RecomputedOldestXmin = GetOldestXmin(NULL, true); if (!TransactionIdPrecedes(OldestXmin, RecomputedOldestXmin)) - record_corrupt_item(items, &tuple.t_data->t_ctid); + record_corrupt_item(items, &tuple.t_self); else { OldestXmin = RecomputedOldestXmin; if (!tuple_all_visible(&tuple, OldestXmin, buffer)) - record_corrupt_item(items, &tuple.t_data->t_ctid); + record_corrupt_item(items, &tuple.t_self); } } @@ -665,7 +679,7 @@ collect_corrupt_items(Oid relid, bool all_visible, bool all_frozen) if (check_frozen) { if (heap_tuple_needs_eventual_freeze(tuple.t_data)) - record_corrupt_item(items, &tuple.t_data->t_ctid); + record_corrupt_item(items, &tuple.t_self); } } From 3b90e38c5d592ea8ec8236287dd5c749fc041728 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 1 Oct 2016 17:15:09 -0400 Subject: [PATCH 259/871] Do ClosePostmasterPorts() earlier in SubPostmasterMain(). In standard Unix builds, postmaster child processes do ClosePostmasterPorts immediately after InitPostmasterChild, that is almost immediately after being spawned. This is important because we don't want children holding open the postmaster's end of the postmaster death watch pipe. However, in EXEC_BACKEND builds, SubPostmasterMain was postponing this responsibility significantly, in order to make it slightly more convenient to pass the right flag value to ClosePostmasterPorts. This is bad, particularly seeing that process_shared_preload_libraries() might invoke nearly-arbitrary code. Rearrange so that we do it as soon as we've fetched the socket FDs via read_backend_variables(). Also move the comment explaining about randomize_va_space to before the call of PGSharedMemoryReAttach, which is where it's relevant. The old placement was appropriate when the reattach happened inside CreateSharedMemoryAndSemaphores, but that was a long time ago. Back-patch to 9.3; the patch doesn't apply cleanly before that, and it doesn't seem worth a lot of effort given that we've had no actual field complaints traceable to this. Discussion: <4157.1475178360@sss.pgh.pa.us> --- src/backend/postmaster/postmaster.c | 57 +++++++++-------------------- 1 file changed, 18 insertions(+), 39 deletions(-) diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 0c0a609735..2d43506cd0 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -4635,10 +4635,17 @@ SubPostmasterMain(int argc, char *argv[]) /* Setup essential subsystems (to ensure elog() behaves sanely) */ InitializeGUCOptions(); + /* Check we got appropriate args */ + if (argc < 3) + elog(FATAL, "invalid subpostmaster invocation"); + /* Read in the variables file */ memset(&port, 0, sizeof(Port)); read_backend_variables(argv[2], &port); + /* Close the postmaster's sockets (as soon as we know them) */ + ClosePostmasterPorts(strcmp(argv[1], "--forklog") == 0); + /* * Set reference point for stack-depth checking */ @@ -4656,15 +4663,21 @@ SubPostmasterMain(int argc, char *argv[]) errmsg("out of memory"))); #endif - /* Check we got appropriate args */ - if (argc < 3) - elog(FATAL, "invalid subpostmaster invocation"); - /* * If appropriate, physically re-attach to shared memory segment. We want * to do this before going any further to ensure that we can attach at the * same address the postmaster used. On the other hand, if we choose not * to re-attach, we may have other cleanup to do. + * + * If testing EXEC_BACKEND on Linux, you should run this as root before + * starting the postmaster: + * + * echo 0 >/proc/sys/kernel/randomize_va_space + * + * This prevents using randomized stack and code addresses that cause the + * child process's memory map to be different from the parent's, making it + * sometimes impossible to attach to shared memory at the desired address. + * Return the setting to its old value (usually '1' or '2') when finished. */ if (strcmp(argv[1], "--forkbackend") == 0 || strcmp(argv[1], "--forkavlauncher") == 0 || @@ -4710,9 +4723,6 @@ SubPostmasterMain(int argc, char *argv[]) { Assert(argc == 3); /* shouldn't be any more args */ - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - /* * Need to reinitialize the SSL library in the backend, since the * context structures contain function pointers and cannot be passed @@ -4743,17 +4753,7 @@ SubPostmasterMain(int argc, char *argv[]) /* Need a PGPROC to run CreateSharedMemoryAndSemaphores */ InitProcess(); - /* - * Attach process to shared data structures. If testing EXEC_BACKEND - * on Linux, you must run this as root before starting the postmaster: - * - * echo 0 >/proc/sys/kernel/randomize_va_space - * - * This prevents a randomized stack base address that causes child - * shared memory to be at a different address than the parent, making - * it impossible to attached to shared memory. Return the value to - * '1' when finished. - */ + /* Attach process to shared data structures */ CreateSharedMemoryAndSemaphores(false, 0); /* And run the backend */ @@ -4761,9 +4761,6 @@ SubPostmasterMain(int argc, char *argv[]) } if (strcmp(argv[1], "--forkboot") == 0) { - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - /* Restore basic shared memory pointers */ InitShmemAccess(UsedShmemSegAddr); @@ -4777,9 +4774,6 @@ SubPostmasterMain(int argc, char *argv[]) } if (strcmp(argv[1], "--forkavlauncher") == 0) { - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - /* Restore basic shared memory pointers */ InitShmemAccess(UsedShmemSegAddr); @@ -4793,9 +4787,6 @@ SubPostmasterMain(int argc, char *argv[]) } if (strcmp(argv[1], "--forkavworker") == 0) { - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - /* Restore basic shared memory pointers */ InitShmemAccess(UsedShmemSegAddr); @@ -4814,9 +4805,6 @@ SubPostmasterMain(int argc, char *argv[]) /* do this as early as possible; in particular, before InitProcess() */ IsBackgroundWorker = true; - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - /* Restore basic shared memory pointers */ InitShmemAccess(UsedShmemSegAddr); @@ -4834,27 +4822,18 @@ SubPostmasterMain(int argc, char *argv[]) } if (strcmp(argv[1], "--forkarch") == 0) { - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - /* Do not want to attach to shared memory */ PgArchiverMain(argc, argv); /* does not return */ } if (strcmp(argv[1], "--forkcol") == 0) { - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - /* Do not want to attach to shared memory */ PgstatCollectorMain(argc, argv); /* does not return */ } if (strcmp(argv[1], "--forklog") == 0) { - /* Close the postmaster's sockets */ - ClosePostmasterPorts(true); - /* Do not want to attach to shared memory */ SysLoggerMain(argc, argv); /* does not return */ From 728ceba938dfadb204a4854bb76ae3b11b635401 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 2 Oct 2016 12:33:46 -0400 Subject: [PATCH 260/871] Avoid leaking FDs after an fsync failure. Fixes errors introduced in commit bc34223bc, as detected by Coverity. In passing, report ENOSPC for a short write while padding a new wal file in open_walfile, make certain that close_walfile closes walfile in all cases, and improve a couple of comments. Michael Paquier and Tom Lane --- src/bin/pg_basebackup/receivelog.c | 56 ++++++++++++++++++++++-------- src/common/file_utils.c | 1 + 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index 8f29d19114..b0fa916b44 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -86,8 +86,11 @@ mark_file_as_archived(const char *basedir, const char *fname, bool do_sync) /* * Open a new WAL file in the specified directory. * - * The file will be padded to 16Mb with zeroes. The base filename (without - * partial_suffix) is stored in current_walfile_name. + * Returns true if OK; on failure, returns false after printing an error msg. + * On success, 'walfile' is set to the FD for the file, and the base filename + * (without partial_suffix) is stored in 'current_walfile_name'. + * + * The file will be padded to 16Mb with zeroes. */ static bool open_walfile(StreamCtl *stream, XLogRecPtr startpoint) @@ -127,18 +130,23 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) } if (statbuf.st_size == XLogSegSize) { - /* File is open and ready to use */ - walfile = f; - /* * fsync, in case of a previous crash between padding and fsyncing the * file. */ - if (stream->do_sync && fsync_fname(fn, false, progname) != 0) - return false; - if (stream->do_sync && fsync_parent_path(fn, progname) != 0) - return false; + if (stream->do_sync) + { + if (fsync_fname(fn, false, progname) != 0 || + fsync_parent_path(fn, progname) != 0) + { + /* error already printed */ + close(f); + return false; + } + } + /* File is open and ready to use */ + walfile = f; return true; } if (statbuf.st_size != 0) @@ -150,12 +158,20 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) return false; } - /* New, empty, file. So pad it to 16Mb with zeroes */ + /* + * New, empty, file. So pad it to 16Mb with zeroes. If we fail partway + * through padding, we should attempt to unlink the file on failure, so as + * not to leave behind a partially-filled file. + */ zerobuf = pg_malloc0(XLOG_BLCKSZ); for (bytes = 0; bytes < XLogSegSize; bytes += XLOG_BLCKSZ) { + errno = 0; if (write(f, zerobuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { + /* if write didn't set errno, assume problem is no disk space */ + if (errno == 0) + errno = ENOSPC; fprintf(stderr, _("%s: could not pad transaction log file \"%s\": %s\n"), progname, fn, strerror(errno)); @@ -173,10 +189,16 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) * using synchronous mode, where the file is modified and fsynced * in-place, without a directory fsync. */ - if (stream->do_sync && fsync_fname(fn, false, progname) != 0) - return false; - if (stream->do_sync && fsync_parent_path(fn, progname) != 0) - return false; + if (stream->do_sync) + { + if (fsync_fname(fn, false, progname) != 0 || + fsync_parent_path(fn, progname) != 0) + { + /* error already printed */ + close(f); + return false; + } + } if (lseek(f, SEEK_SET, 0) != 0) { @@ -186,6 +208,8 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) close(f); return false; } + + /* File is open and ready to use */ walfile = f; return true; } @@ -209,6 +233,8 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) fprintf(stderr, _("%s: could not determine seek position in file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); + close(walfile); + walfile = -1; return false; } @@ -216,6 +242,8 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); + close(walfile); + walfile = -1; return false; } diff --git a/src/common/file_utils.c b/src/common/file_utils.c index 1d855645b9..1855e2372c 100644 --- a/src/common/file_utils.c +++ b/src/common/file_utils.c @@ -273,6 +273,7 @@ fsync_fname(const char *fname, bool isdir, const char *progname) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, fname, strerror(errno)); + (void) close(fd); return -1; } From e8bdee2770ff52aab208bc6f8965a4a01979d0aa Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 2 Oct 2016 14:31:28 -0400 Subject: [PATCH 261/871] Add ALTER EXTENSION ADD/DROP ACCESS METHOD, and use it in pg_upgrade. Without this, an extension containing an access method is not properly dumped/restored during pg_upgrade --- the AM ends up not being a member of the extension after upgrading. Another oversight in commit 473b93287, reported by Andrew Dunstan. Report: --- doc/src/sgml/ref/alter_extension.sgml | 1 + src/backend/parser/gram.y | 11 ++++++++++- src/bin/pg_dump/pg_dump.c | 3 +++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/alter_extension.sgml b/doc/src/sgml/ref/alter_extension.sgml index 7141ee352e..de6d6dca16 100644 --- a/doc/src/sgml/ref/alter_extension.sgml +++ b/doc/src/sgml/ref/alter_extension.sgml @@ -30,6 +30,7 @@ ALTER EXTENSION name DROP where member_object is: + ACCESS METHOD object_name | AGGREGATE aggregate_name ( aggregate_signature ) | CAST (source_type AS target_type) | COLLATION object_name | diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 1526c73a1c..5547fc8658 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -3931,7 +3931,16 @@ alter_extension_opt_item: *****************************************************************************/ AlterExtensionContentsStmt: - ALTER EXTENSION name add_drop AGGREGATE func_name aggr_args + ALTER EXTENSION name add_drop ACCESS METHOD name + { + AlterExtensionContentsStmt *n = makeNode(AlterExtensionContentsStmt); + n->extname = $3; + n->action = $4; + n->objtype = OBJECT_ACCESS_METHOD; + n->objname = list_make1(makeString($7)); + $$ = (Node *)n; + } + | ALTER EXTENSION name add_drop AGGREGATE func_name aggr_args { AlterExtensionContentsStmt *n = makeNode(AlterExtensionContentsStmt); n->extname = $3; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 51b8a1a622..299e88788e 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -12505,6 +12505,9 @@ dumpAccessMethod(Archive *fout, AccessMethodInfo *aminfo) appendPQExpBuffer(labelq, "ACCESS METHOD %s", qamname); + if (dopt->binary_upgrade) + binary_upgrade_extension_member(q, &aminfo->dobj, labelq->data); + if (aminfo->dobj.dump & DUMP_COMPONENT_DEFINITION) ArchiveEntry(fout, aminfo->dobj.catId, aminfo->dobj.dumpId, aminfo->dobj.name, From e94568ecc10f2638e542ae34f2990b821bbf90ac Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 3 Oct 2016 13:37:49 +0300 Subject: [PATCH 262/871] Change the way pre-reading in external sort's merge phase works. Don't pre-read tuples into SortTuple slots during merge. Instead, use the memory for larger read buffers in logtape.c. We're doing the same number of READTUP() calls either way, but managing the pre-read SortTuple slots is much more complicated. Also, the on-tape representation is more compact than SortTuples, so we can fit more pre-read tuples into the same amount of memory this way. And we have better cache-locality, when we use just a small number of SortTuple slots. Now that we only hold one tuple from each tape in the SortTuple slots, we can greatly simplify the "batch memory" management. We now maintain a small set of fixed-sized slots, to hold the tuples, and fall back to palloc() for larger tuples. We use this method during all merge phases, not just the final merge, and also when randomAccess is requested, and also in the TSS_SORTEDONTAPE case. In other words, it's used whenever we do an external sort. Reviewed by Peter Geoghegan and Claudio Freire. Discussion: --- src/backend/utils/sort/logtape.c | 153 +++- src/backend/utils/sort/tuplesort.c | 1216 ++++++++++------------------ src/include/utils/logtape.h | 2 + 3 files changed, 574 insertions(+), 797 deletions(-) diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index 774520752f..caa6960b95 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -52,12 +52,17 @@ * not clear this helps much, but it can't hurt. (XXX perhaps a LIFO * policy for free blocks would be better?) * + * To further make the I/Os more sequential, we can use a larger buffer + * when reading, and read multiple blocks from the same tape in one go, + * whenever the buffer becomes empty. LogicalTapeAssignReadBufferSize() + * can be used to set the size of the read buffer. + * * To support the above policy of writing to the lowest free block, * ltsGetFreeBlock sorts the list of free block numbers into decreasing * order each time it is asked for a block and the list isn't currently * sorted. This is an efficient way to handle it because we expect cycles * of releasing many blocks followed by re-using many blocks, due to - * tuplesort.c's "preread" behavior. + * the larger read buffer. * * Since all the bookkeeping and buffer memory is allocated with palloc(), * and the underlying file(s) are made with OpenTemporaryFile, all resources @@ -79,6 +84,7 @@ #include "storage/buffile.h" #include "utils/logtape.h" +#include "utils/memutils.h" /* * Block indexes are "long"s, so we can fit this many per indirect block. @@ -131,9 +137,18 @@ typedef struct LogicalTape * reading. */ char *buffer; /* physical buffer (separately palloc'd) */ + int buffer_size; /* allocated size of the buffer */ long curBlockNumber; /* this block's logical blk# within tape */ int pos; /* next read/write position in buffer */ int nbytes; /* total # of valid bytes in buffer */ + + /* + * Desired buffer size to use when reading. To keep things simple, we use + * a single-block buffer when writing, or when reading a frozen tape. But + * when we are reading and will only read forwards, we allocate a larger + * buffer, determined by read_buffer_size. + */ + int read_buffer_size; } LogicalTape; /* @@ -227,6 +242,53 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer) blocknum))); } +/* + * Read as many blocks as we can into the per-tape buffer. + * + * The caller can specify the next physical block number to read, in + * datablocknum, or -1 to fetch the next block number from the internal block. + * If datablocknum == -1, the caller must've already set curBlockNumber. + * + * Returns true if anything was read, 'false' on EOF. + */ +static bool +ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt, long datablocknum) +{ + lt->pos = 0; + lt->nbytes = 0; + + do + { + /* Fetch next block number (unless provided by caller) */ + if (datablocknum == -1) + { + datablocknum = ltsRecallNextBlockNum(lts, lt->indirect, lt->frozen); + if (datablocknum == -1L) + break; /* EOF */ + lt->curBlockNumber++; + } + + /* Read the block */ + ltsReadBlock(lts, datablocknum, (void *) (lt->buffer + lt->nbytes)); + if (!lt->frozen) + ltsReleaseBlock(lts, datablocknum); + + if (lt->curBlockNumber < lt->numFullBlocks) + lt->nbytes += BLCKSZ; + else + { + /* EOF */ + lt->nbytes += lt->lastBlockBytes; + break; + } + + /* Advance to next block, if we have buffer space left */ + datablocknum = -1; + } while (lt->nbytes < lt->buffer_size); + + return (lt->nbytes > 0); +} + /* * qsort comparator for sorting freeBlocks[] into decreasing order. */ @@ -546,6 +608,8 @@ LogicalTapeSetCreate(int ntapes) lt->numFullBlocks = 0L; lt->lastBlockBytes = 0; lt->buffer = NULL; + lt->buffer_size = 0; + lt->read_buffer_size = BLCKSZ; lt->curBlockNumber = 0L; lt->pos = 0; lt->nbytes = 0; @@ -628,7 +692,10 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum, /* Allocate data buffer and first indirect block on first write */ if (lt->buffer == NULL) + { lt->buffer = (char *) palloc(BLCKSZ); + lt->buffer_size = BLCKSZ; + } if (lt->indirect == NULL) { lt->indirect = (IndirectBlock *) palloc(sizeof(IndirectBlock)); @@ -636,6 +703,7 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum, lt->indirect->nextup = NULL; } + Assert(lt->buffer_size == BLCKSZ); while (size > 0) { if (lt->pos >= BLCKSZ) @@ -709,18 +777,19 @@ LogicalTapeRewind(LogicalTapeSet *lts, int tapenum, bool forWrite) Assert(lt->frozen); datablocknum = ltsRewindFrozenIndirectBlock(lts, lt->indirect); } + + /* Allocate a read buffer */ + if (lt->buffer) + pfree(lt->buffer); + lt->buffer = palloc(lt->read_buffer_size); + lt->buffer_size = lt->read_buffer_size; + /* Read the first block, or reset if tape is empty */ lt->curBlockNumber = 0L; lt->pos = 0; lt->nbytes = 0; if (datablocknum != -1L) - { - ltsReadBlock(lts, datablocknum, (void *) lt->buffer); - if (!lt->frozen) - ltsReleaseBlock(lts, datablocknum); - lt->nbytes = (lt->curBlockNumber < lt->numFullBlocks) ? - BLCKSZ : lt->lastBlockBytes; - } + ltsReadFillBuffer(lts, lt, datablocknum); } else { @@ -754,6 +823,13 @@ LogicalTapeRewind(LogicalTapeSet *lts, int tapenum, bool forWrite) lt->curBlockNumber = 0L; lt->pos = 0; lt->nbytes = 0; + + if (lt->buffer) + { + pfree(lt->buffer); + lt->buffer = NULL; + lt->buffer_size = 0; + } } } @@ -779,20 +855,8 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum, if (lt->pos >= lt->nbytes) { /* Try to load more data into buffer. */ - long datablocknum = ltsRecallNextBlockNum(lts, lt->indirect, - lt->frozen); - - if (datablocknum == -1L) + if (!ltsReadFillBuffer(lts, lt, -1)) break; /* EOF */ - lt->curBlockNumber++; - lt->pos = 0; - ltsReadBlock(lts, datablocknum, (void *) lt->buffer); - if (!lt->frozen) - ltsReleaseBlock(lts, datablocknum); - lt->nbytes = (lt->curBlockNumber < lt->numFullBlocks) ? - BLCKSZ : lt->lastBlockBytes; - if (lt->nbytes <= 0) - break; /* EOF (possible here?) */ } nthistime = lt->nbytes - lt->pos; @@ -842,6 +906,22 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum) lt->writing = false; lt->frozen = true; datablocknum = ltsRewindIndirectBlock(lts, lt->indirect, true); + + /* + * The seek and backspace functions assume a single block read buffer. + * That's OK with current usage. A larger buffer is helpful to make the + * read pattern of the backing file look more sequential to the OS, when + * we're reading from multiple tapes. But at the end of a sort, when a + * tape is frozen, we only read from a single tape anyway. + */ + if (!lt->buffer || lt->buffer_size != BLCKSZ) + { + if (lt->buffer) + pfree(lt->buffer); + lt->buffer = palloc(BLCKSZ); + lt->buffer_size = BLCKSZ; + } + /* Read the first block, or reset if tape is empty */ lt->curBlockNumber = 0L; lt->pos = 0; @@ -875,6 +955,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size) Assert(tapenum >= 0 && tapenum < lts->nTapes); lt = <s->tapes[tapenum]; Assert(lt->frozen); + Assert(lt->buffer_size == BLCKSZ); /* * Easy case for seek within current block. @@ -941,6 +1022,7 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum, lt = <s->tapes[tapenum]; Assert(lt->frozen); Assert(offset >= 0 && offset <= BLCKSZ); + Assert(lt->buffer_size == BLCKSZ); /* * Easy case for seek within current block. @@ -1002,6 +1084,10 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum, Assert(tapenum >= 0 && tapenum < lts->nTapes); lt = <s->tapes[tapenum]; + + /* With a larger buffer, 'pos' wouldn't be the same as offset within page */ + Assert(lt->buffer_size == BLCKSZ); + *blocknum = lt->curBlockNumber; *offset = lt->pos; } @@ -1014,3 +1100,28 @@ LogicalTapeSetBlocks(LogicalTapeSet *lts) { return lts->nFileBlocks; } + +/* + * Set buffer size to use, when reading from given tape. + */ +void +LogicalTapeAssignReadBufferSize(LogicalTapeSet *lts, int tapenum, size_t avail_mem) +{ + LogicalTape *lt; + + Assert(tapenum >= 0 && tapenum < lts->nTapes); + lt = <s->tapes[tapenum]; + + /* + * The buffer size must be a multiple of BLCKSZ in size, so round the + * given value down to nearest BLCKSZ. Make sure we have at least one + * page. Also, don't go above MaxAllocSize, to avoid erroring out. A + * multi-gigabyte buffer is unlikely to be helpful, anyway. + */ + if (avail_mem < BLCKSZ) + avail_mem = BLCKSZ; + if (avail_mem > MaxAllocSize) + avail_mem = MaxAllocSize; + avail_mem -= avail_mem % BLCKSZ; + lt->read_buffer_size = avail_mem; +} diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index 16ceb30b27..20cfb0b139 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -74,7 +74,7 @@ * the merge is complete. The basic merge algorithm thus needs very little * memory --- only M tuples for an M-way merge, and M is constrained to a * small number. However, we can still make good use of our full workMem - * allocation by pre-reading additional tuples from each source tape. Without + * allocation by pre-reading additional blocks from each source tape. Without * prereading, our access pattern to the temporary file would be very erratic; * on average we'd read one block from each of M source tapes during the same * time that we're writing M blocks to the output tape, so there is no @@ -84,10 +84,10 @@ * worse when it comes time to read that tape. A straightforward merge pass * thus ends up doing a lot of waiting for disk seeks. We can improve matters * by prereading from each source tape sequentially, loading about workMem/M - * bytes from each tape in turn. Then we run the merge algorithm, writing but - * not reading until one of the preloaded tuple series runs out. Then we - * switch back to preread mode, fill memory again, and repeat. This approach - * helps to localize both read and write accesses. + * bytes from each tape in turn, and making the sequential blocks immediately + * available for reuse. This approach helps to localize both read and write + * accesses. The pre-reading is handled by logtape.c, we just tell it how + * much memory to use for the buffers. * * When the caller requests random access to the sort result, we form * the final sorted run on a logical tape which is then "frozen", so @@ -162,9 +162,9 @@ bool optimize_bounded_sort = true; * The objects we actually sort are SortTuple structs. These contain * a pointer to the tuple proper (might be a MinimalTuple or IndexTuple), * which is a separate palloc chunk --- we assume it is just one chunk and - * can be freed by a simple pfree() (except during final on-the-fly merge, - * when memory is used in batch). SortTuples also contain the tuple's - * first key column in Datum/nullflag format, and an index integer. + * can be freed by a simple pfree() (except during merge, when we use a + * simple slab allocator). SortTuples also contain the tuple's first key + * column in Datum/nullflag format, and an index integer. * * Storing the first key column lets us save heap_getattr or index_getattr * calls during tuple comparisons. We could extract and save all the key @@ -191,9 +191,8 @@ bool optimize_bounded_sort = true; * it now only distinguishes RUN_FIRST and HEAP_RUN_NEXT, since replacement * selection is always abandoned after the first run; no other run number * should be represented here. During merge passes, we re-use it to hold the - * input tape number that each tuple in the heap was read from, or to hold the - * index of the next tuple pre-read from the same tape in the case of pre-read - * entries. tupindex goes unused if the sort occurs entirely in memory. + * input tape number that each tuple in the heap was read from. tupindex goes + * unused if the sort occurs entirely in memory. */ typedef struct { @@ -203,6 +202,24 @@ typedef struct int tupindex; /* see notes above */ } SortTuple; +/* + * During merge, we use a pre-allocated set of fixed-size slots to hold + * tuples. To avoid palloc/pfree overhead. + * + * Merge doesn't require a lot of memory, so we can afford to waste some, + * by using gratuitously-sized slots. If a tuple is larger than 1 kB, the + * palloc() overhead is not significant anymore. + * + * 'nextfree' is valid when this chunk is in the free list. When in use, the + * slot holds a tuple. + */ +#define SLAB_SLOT_SIZE 1024 + +typedef union SlabSlot +{ + union SlabSlot *nextfree; + char buffer[SLAB_SLOT_SIZE]; +} SlabSlot; /* * Possible states of a Tuplesort object. These denote the states that @@ -288,41 +305,28 @@ struct Tuplesortstate /* * Function to write a stored tuple onto tape. The representation of the * tuple on tape need not be the same as it is in memory; requirements on - * the tape representation are given below. After writing the tuple, - * pfree() the out-of-line data (not the SortTuple struct!), and increase - * state->availMem by the amount of memory space thereby released. + * the tape representation are given below. Unless the slab allocator is + * used, after writing the tuple, pfree() the out-of-line data (not the + * SortTuple struct!), and increase state->availMem by the amount of + * memory space thereby released. */ void (*writetup) (Tuplesortstate *state, int tapenum, SortTuple *stup); /* * Function to read a stored tuple from tape back into memory. 'len' is - * the already-read length of the stored tuple. Create a palloc'd copy, - * initialize tuple/datum1/isnull1 in the target SortTuple struct, and - * decrease state->availMem by the amount of memory space consumed. (See - * batchUsed notes for details on how memory is handled when incremental - * accounting is abandoned.) + * the already-read length of the stored tuple. The tuple is allocated + * from the slab memory arena, or is palloc'd, see readtup_alloc(). */ void (*readtup) (Tuplesortstate *state, SortTuple *stup, int tapenum, unsigned int len); - /* - * Function to move a caller tuple. This is usually implemented as a - * memmove() shim, but function may also perform additional fix-up of - * caller tuple where needed. Batch memory support requires the movement - * of caller tuples from one location in memory to another. - */ - void (*movetup) (void *dest, void *src, unsigned int len); - /* * This array holds the tuples now in sort memory. If we are in state * INITIAL, the tuples are in no particular order; if we are in state * SORTEDINMEM, the tuples are in final sorted order; in states BUILDRUNS * and FINALMERGE, the tuples are organized in "heap" order per Algorithm - * H. (Note that memtupcount only counts the tuples that are part of the - * heap --- during merge passes, memtuples[] entries beyond tapeRange are - * never in the heap and are used to hold pre-read tuples.) In state - * SORTEDONTAPE, the array is not used. + * H. In state SORTEDONTAPE, the array is not used. */ SortTuple *memtuples; /* array of SortTuple structs */ int memtupcount; /* number of tuples currently present */ @@ -330,13 +334,45 @@ struct Tuplesortstate bool growmemtuples; /* memtuples' growth still underway? */ /* - * Memory for tuples is sometimes allocated in batch, rather than - * incrementally. This implies that incremental memory accounting has - * been abandoned. Currently, this only happens for the final on-the-fly - * merge step. Large batch allocations can store tuples (e.g. - * IndexTuples) without palloc() fragmentation and other overhead. + * Memory for tuples is sometimes allocated using a simple slab allocator, + * rather than with palloc(). Currently, we switch to slab allocation + * when we start merging. Merging only needs to keep a small, fixed + * number of tuples in memory at any time, so we can avoid the + * palloc/pfree overhead by recycling a fixed number of fixed-size slots + * to hold the tuples. + * + * For the slab, we use one large allocation, divided into SLAB_SLOT_SIZE + * slots. The allocation is sized to have one slot per tape, plus one + * additional slot. We need that many slots to hold all the tuples kept + * in the heap during merge, plus the one we have last returned from the + * sort, with tuplesort_gettuple. + * + * Initially, all the slots are kept in a linked list of free slots. When + * a tuple is read from a tape, it is put to the next available slot, if + * it fits. If the tuple is larger than SLAB_SLOT_SIZE, it is palloc'd + * instead. + * + * When we're done processing a tuple, we return the slot back to the free + * list, or pfree() if it was palloc'd. We know that a tuple was + * allocated from the slab, if its pointer value is between + * slabMemoryBegin and -End. + * + * When the slab allocator is used, the USEMEM/LACKMEM mechanism of + * tracking memory usage is not used. + */ + bool slabAllocatorUsed; + + char *slabMemoryBegin; /* beginning of slab memory arena */ + char *slabMemoryEnd; /* end of slab memory arena */ + SlabSlot *slabFreeHead; /* head of free list */ + + /* + * When we return a tuple to the caller in tuplesort_gettuple_XXX, that + * came from a tape (that is, in TSS_SORTEDONTAPE or TSS_FINALMERGE + * modes), we remember the tuple in 'lastReturnedTuple', so that we can + * recycle the memory on next gettuple call. */ - bool batchUsed; + void *lastReturnedTuple; /* * While building initial runs, this indicates if the replacement @@ -358,42 +394,11 @@ struct Tuplesortstate */ /* - * These variables are only used during merge passes. mergeactive[i] is - * true if we are reading an input run from (actual) tape number i and - * have not yet exhausted that run. mergenext[i] is the memtuples index - * of the next pre-read tuple (next to be loaded into the heap) for tape - * i, or 0 if we are out of pre-read tuples. mergelast[i] similarly - * points to the last pre-read tuple from each tape. mergeavailslots[i] - * is the number of unused memtuples[] slots reserved for tape i, and - * mergeavailmem[i] is the amount of unused space allocated for tape i. - * mergefreelist and mergefirstfree keep track of unused locations in the - * memtuples[] array. The memtuples[].tupindex fields link together - * pre-read tuples for each tape as well as recycled locations in - * mergefreelist. It is OK to use 0 as a null link in these lists, because - * memtuples[0] is part of the merge heap and is never a pre-read tuple. + * This variable is only used during merge passes. mergeactive[i] is true + * if we are reading an input run from (actual) tape number i and have not + * yet exhausted that run. */ bool *mergeactive; /* active input run source? */ - int *mergenext; /* first preread tuple for each source */ - int *mergelast; /* last preread tuple for each source */ - int *mergeavailslots; /* slots left for prereading each tape */ - int64 *mergeavailmem; /* availMem for prereading each tape */ - int mergefreelist; /* head of freelist of recycled slots */ - int mergefirstfree; /* first slot never used in this merge */ - - /* - * Per-tape batch state, when final on-the-fly merge consumes memory from - * just a few large allocations. - * - * Aside from the general benefits of performing fewer individual retail - * palloc() calls, this also helps make merging more cache efficient, - * since each tape's tuples must naturally be accessed sequentially (in - * sorted order). - */ - int64 spacePerTape; /* Space (memory) for tuples (not slots) */ - char **mergetuples; /* Each tape's memory allocation */ - char **mergecurrent; /* Current offset into each tape's memory */ - char **mergetail; /* Last item's start point for each tape */ - char **mergeoverflow; /* Retail palloc() "overflow" for each tape */ /* * Variables for Algorithm D. Note that destTape is a "logical" tape @@ -481,12 +486,34 @@ struct Tuplesortstate #endif }; +/* + * Is the given tuple allocated from the slab memory arena? + */ +#define IS_SLAB_SLOT(state, tuple) \ + ((char *) (tuple) >= (state)->slabMemoryBegin && \ + (char *) (tuple) < (state)->slabMemoryEnd) + +/* + * Return the given tuple to the slab memory free list, or free it + * if it was palloc'd. + */ +#define RELEASE_SLAB_SLOT(state, tuple) \ + do { \ + SlabSlot *buf = (SlabSlot *) tuple; \ + \ + if (IS_SLAB_SLOT((state), buf)) \ + { \ + buf->nextfree = (state)->slabFreeHead; \ + (state)->slabFreeHead = buf; \ + } else \ + pfree(buf); \ + } while(0) + #define COMPARETUP(state,a,b) ((*(state)->comparetup) (a, b, state)) #define COPYTUP(state,stup,tup) ((*(state)->copytup) (state, stup, tup)) #define WRITETUP(state,tape,stup) ((*(state)->writetup) (state, tape, stup)) #define READTUP(state,stup,tape,len) ((*(state)->readtup) (state, stup, tape, len)) -#define MOVETUP(dest,src,len) ((*(state)->movetup) (dest, src, len)) -#define LACKMEM(state) ((state)->availMem < 0 && !(state)->batchUsed) +#define LACKMEM(state) ((state)->availMem < 0 && !(state)->slabAllocatorUsed) #define USEMEM(state,amt) ((state)->availMem -= (amt)) #define FREEMEM(state,amt) ((state)->availMem += (amt)) @@ -551,18 +578,12 @@ static bool consider_abort_common(Tuplesortstate *state); static bool useselection(Tuplesortstate *state); static void inittapes(Tuplesortstate *state); static void selectnewtape(Tuplesortstate *state); +static void init_slab_allocator(Tuplesortstate *state, int numSlots); +static void init_tape_buffers(Tuplesortstate *state, int numInputTapes); static void mergeruns(Tuplesortstate *state); static void mergeonerun(Tuplesortstate *state); -static void beginmerge(Tuplesortstate *state, bool finalMergeBatch); -static void batchmemtuples(Tuplesortstate *state); -static void mergebatch(Tuplesortstate *state, int64 spacePerTape); -static void mergebatchone(Tuplesortstate *state, int srcTape, - SortTuple *stup, bool *should_free); -static void mergebatchfreetape(Tuplesortstate *state, int srcTape, - SortTuple *rtup, bool *should_free); -static void *mergebatchalloc(Tuplesortstate *state, int tapenum, Size tuplen); -static void mergepreread(Tuplesortstate *state); -static void mergeprereadone(Tuplesortstate *state, int srcTape); +static void beginmerge(Tuplesortstate *state); +static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup); static void dumptuples(Tuplesortstate *state, bool alltuples); static void dumpbatch(Tuplesortstate *state, bool alltuples); static void make_bounded_heap(Tuplesortstate *state); @@ -576,7 +597,7 @@ static void tuplesort_heap_delete_top(Tuplesortstate *state, bool checkIndex); static void reversedirection(Tuplesortstate *state); static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK); static void markrunend(Tuplesortstate *state, int tapenum); -static void *readtup_alloc(Tuplesortstate *state, int tapenum, Size tuplen); +static void *readtup_alloc(Tuplesortstate *state, Size tuplen); static int comparetup_heap(const SortTuple *a, const SortTuple *b, Tuplesortstate *state); static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup); @@ -584,7 +605,6 @@ static void writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup); static void readtup_heap(Tuplesortstate *state, SortTuple *stup, int tapenum, unsigned int len); -static void movetup_heap(void *dest, void *src, unsigned int len); static int comparetup_cluster(const SortTuple *a, const SortTuple *b, Tuplesortstate *state); static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup); @@ -592,7 +612,6 @@ static void writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup); static void readtup_cluster(Tuplesortstate *state, SortTuple *stup, int tapenum, unsigned int len); -static void movetup_cluster(void *dest, void *src, unsigned int len); static int comparetup_index_btree(const SortTuple *a, const SortTuple *b, Tuplesortstate *state); static int comparetup_index_hash(const SortTuple *a, const SortTuple *b, @@ -602,7 +621,6 @@ static void writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup); static void readtup_index(Tuplesortstate *state, SortTuple *stup, int tapenum, unsigned int len); -static void movetup_index(void *dest, void *src, unsigned int len); static int comparetup_datum(const SortTuple *a, const SortTuple *b, Tuplesortstate *state); static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup); @@ -610,7 +628,6 @@ static void writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup); static void readtup_datum(Tuplesortstate *state, SortTuple *stup, int tapenum, unsigned int len); -static void movetup_datum(void *dest, void *src, unsigned int len); static void free_sort_tuple(Tuplesortstate *state, SortTuple *stup); /* @@ -705,7 +722,7 @@ tuplesort_begin_common(int workMem, bool randomAccess) ALLOCSET_SEPARATE_THRESHOLD / sizeof(SortTuple) + 1); state->growmemtuples = true; - state->batchUsed = false; + state->slabAllocatorUsed = false; state->memtuples = (SortTuple *) palloc(state->memtupsize * sizeof(SortTuple)); USEMEM(state, GetMemoryChunkSpace(state->memtuples)); @@ -762,7 +779,6 @@ tuplesort_begin_heap(TupleDesc tupDesc, state->copytup = copytup_heap; state->writetup = writetup_heap; state->readtup = readtup_heap; - state->movetup = movetup_heap; state->tupDesc = tupDesc; /* assume we need not copy tupDesc */ state->abbrevNext = 10; @@ -835,7 +851,6 @@ tuplesort_begin_cluster(TupleDesc tupDesc, state->copytup = copytup_cluster; state->writetup = writetup_cluster; state->readtup = readtup_cluster; - state->movetup = movetup_cluster; state->abbrevNext = 10; state->indexInfo = BuildIndexInfo(indexRel); @@ -927,7 +942,6 @@ tuplesort_begin_index_btree(Relation heapRel, state->copytup = copytup_index; state->writetup = writetup_index; state->readtup = readtup_index; - state->movetup = movetup_index; state->abbrevNext = 10; state->heapRel = heapRel; @@ -995,7 +1009,6 @@ tuplesort_begin_index_hash(Relation heapRel, state->copytup = copytup_index; state->writetup = writetup_index; state->readtup = readtup_index; - state->movetup = movetup_index; state->heapRel = heapRel; state->indexRel = indexRel; @@ -1038,7 +1051,6 @@ tuplesort_begin_datum(Oid datumType, Oid sortOperator, Oid sortCollation, state->copytup = copytup_datum; state->writetup = writetup_datum; state->readtup = readtup_datum; - state->movetup = movetup_datum; state->abbrevNext = 10; state->datumType = datumType; @@ -1838,7 +1850,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward, { case TSS_SORTEDINMEM: Assert(forward || state->randomAccess); - Assert(!state->batchUsed); + Assert(!state->slabAllocatorUsed); *should_free = false; if (forward) { @@ -1883,15 +1895,35 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward, case TSS_SORTEDONTAPE: Assert(forward || state->randomAccess); - Assert(!state->batchUsed); - *should_free = true; + Assert(state->slabAllocatorUsed); + + /* + * The slot that held the tuple that we returned in previous + * gettuple call can now be reused. + */ + if (state->lastReturnedTuple) + { + RELEASE_SLAB_SLOT(state, state->lastReturnedTuple); + state->lastReturnedTuple = NULL; + } + if (forward) { if (state->eof_reached) return false; + if ((tuplen = getlen(state, state->result_tape, true)) != 0) { READTUP(state, stup, state->result_tape, tuplen); + + /* + * Remember the tuple we return, so that we can recycle + * its memory on next call. (This can be NULL, in the + * !state->tuples case). + */ + state->lastReturnedTuple = stup->tuple; + + *should_free = false; return true; } else @@ -1965,74 +1997,70 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward, tuplen)) elog(ERROR, "bogus tuple length in backward scan"); READTUP(state, stup, state->result_tape, tuplen); + + /* + * Remember the tuple we return, so that we can recycle its memory + * on next call. (This can be NULL, in the Datum case). + */ + state->lastReturnedTuple = stup->tuple; + + *should_free = false; return true; case TSS_FINALMERGE: Assert(forward); - Assert(state->batchUsed || !state->tuples); - /* For now, assume tuple is stored in tape's batch memory */ + /* We are managing memory ourselves, with the slab allocator. */ + Assert(state->slabAllocatorUsed); *should_free = false; + /* + * The slab slot holding the tuple that we returned in previous + * gettuple call can now be reused. + */ + if (state->lastReturnedTuple) + { + RELEASE_SLAB_SLOT(state, state->lastReturnedTuple); + state->lastReturnedTuple = NULL; + } + /* * This code should match the inner loop of mergeonerun(). */ if (state->memtupcount > 0) { int srcTape = state->memtuples[0].tupindex; - int tupIndex; - SortTuple *newtup; + SortTuple newtup; + + *stup = state->memtuples[0]; /* - * Returned tuple is still counted in our memory space most of - * the time. See mergebatchone() for discussion of why caller - * may occasionally be required to free returned tuple, and - * how preread memory is managed with regard to edge cases - * more generally. + * Remember the tuple we return, so that we can recycle its + * memory on next call. (This can be NULL, in the Datum case). */ - *stup = state->memtuples[0]; - if ((tupIndex = state->mergenext[srcTape]) == 0) + state->lastReturnedTuple = stup->tuple; + + /* + * Pull next tuple from tape, and replace the returned tuple + * at top of the heap with it. + */ + if (!mergereadnext(state, srcTape, &newtup)) { /* - * out of preloaded data on this tape, try to read more - * - * Unlike mergeonerun(), we only preload from the single - * tape that's run dry, though not before preparing its - * batch memory for a new round of sequential consumption. - * See mergepreread() comments. + * If no more data, we've reached end of run on this tape. + * Remove the top node from the heap. */ - if (state->batchUsed) - mergebatchone(state, srcTape, stup, should_free); - - mergeprereadone(state, srcTape); + tuplesort_heap_delete_top(state, false); /* - * if still no data, we've reached end of run on this tape + * Rewind to free the read buffer. It'd go away at the + * end of the sort anyway, but better to release the + * memory early. */ - if ((tupIndex = state->mergenext[srcTape]) == 0) - { - /* Remove the top node from the heap */ - tuplesort_heap_delete_top(state, false); - /* Free tape's buffer, avoiding dangling pointer */ - if (state->batchUsed) - mergebatchfreetape(state, srcTape, stup, should_free); - return true; - } + LogicalTapeRewind(state->tapeset, srcTape, true); + return true; } - - /* - * pull next preread tuple from list, and replace the returned - * tuple at top of the heap with it. - */ - newtup = &state->memtuples[tupIndex]; - state->mergenext[srcTape] = newtup->tupindex; - if (state->mergenext[srcTape] == 0) - state->mergelast[srcTape] = 0; - newtup->tupindex = srcTape; - tuplesort_heap_replace_top(state, newtup, false); - /* put the now-unused memtuples entry on the freelist */ - newtup->tupindex = state->mergefreelist; - state->mergefreelist = tupIndex; - state->mergeavailslots[srcTape]++; + newtup.tupindex = srcTape; + tuplesort_heap_replace_top(state, &newtup, false); return true; } return false; @@ -2317,13 +2345,6 @@ inittapes(Tuplesortstate *state) /* Compute number of tapes to use: merge order plus 1 */ maxTapes = tuplesort_merge_order(state->allowedMem) + 1; - /* - * We must have at least 2*maxTapes slots in the memtuples[] array, else - * we'd not have room for merge heap plus preread. It seems unlikely that - * this case would ever occur, but be safe. - */ - maxTapes = Min(maxTapes, state->memtupsize / 2); - state->maxTapes = maxTapes; state->tapeRange = maxTapes - 1; @@ -2334,13 +2355,13 @@ inittapes(Tuplesortstate *state) #endif /* - * Decrease availMem to reflect the space needed for tape buffers; but - * don't decrease it to the point that we have no room for tuples. (That - * case is only likely to occur if sorting pass-by-value Datums; in all - * other scenarios the memtuples[] array is unlikely to occupy more than - * half of allowedMem. In the pass-by-value case it's not important to - * account for tuple space, so we don't care if LACKMEM becomes - * inaccurate.) + * Decrease availMem to reflect the space needed for tape buffers, when + * writing the initial runs; but don't decrease it to the point that we + * have no room for tuples. (That case is only likely to occur if sorting + * pass-by-value Datums; in all other scenarios the memtuples[] array is + * unlikely to occupy more than half of allowedMem. In the pass-by-value + * case it's not important to account for tuple space, so we don't care if + * LACKMEM becomes inaccurate.) */ tapeSpace = (int64) maxTapes *TAPE_BUFFER_OVERHEAD; @@ -2359,14 +2380,6 @@ inittapes(Tuplesortstate *state) state->tapeset = LogicalTapeSetCreate(maxTapes); state->mergeactive = (bool *) palloc0(maxTapes * sizeof(bool)); - state->mergenext = (int *) palloc0(maxTapes * sizeof(int)); - state->mergelast = (int *) palloc0(maxTapes * sizeof(int)); - state->mergeavailslots = (int *) palloc0(maxTapes * sizeof(int)); - state->mergeavailmem = (int64 *) palloc0(maxTapes * sizeof(int64)); - state->mergetuples = (char **) palloc0(maxTapes * sizeof(char *)); - state->mergecurrent = (char **) palloc0(maxTapes * sizeof(char *)); - state->mergetail = (char **) palloc0(maxTapes * sizeof(char *)); - state->mergeoverflow = (char **) palloc0(maxTapes * sizeof(char *)); state->tp_fib = (int *) palloc0(maxTapes * sizeof(int)); state->tp_runs = (int *) palloc0(maxTapes * sizeof(int)); state->tp_dummy = (int *) palloc0(maxTapes * sizeof(int)); @@ -2465,6 +2478,105 @@ selectnewtape(Tuplesortstate *state) state->destTape = 0; } +/* + * Initialize the slab allocation arena, for the given number of slots. + */ +static void +init_slab_allocator(Tuplesortstate *state, int numSlots) +{ + if (numSlots > 0) + { + char *p; + int i; + + state->slabMemoryBegin = palloc(numSlots * SLAB_SLOT_SIZE); + state->slabMemoryEnd = state->slabMemoryBegin + + numSlots * SLAB_SLOT_SIZE; + state->slabFreeHead = (SlabSlot *) state->slabMemoryBegin; + USEMEM(state, numSlots * SLAB_SLOT_SIZE); + + p = state->slabMemoryBegin; + for (i = 0; i < numSlots - 1; i++) + { + ((SlabSlot *) p)->nextfree = (SlabSlot *) (p + SLAB_SLOT_SIZE); + p += SLAB_SLOT_SIZE; + } + ((SlabSlot *) p)->nextfree = NULL; + } + else + { + state->slabMemoryBegin = state->slabMemoryEnd = NULL; + state->slabFreeHead = NULL; + } + state->slabAllocatorUsed = true; +} + +/* + * Divide all remaining work memory (availMem) as read buffers, for all + * the tapes that will be used during the merge. + * + * We use the number of possible *input* tapes here, rather than maxTapes, + * for the calculation. At all times, we'll be reading from at most + * numInputTapes tapes, and one tape is used for output (unless we do an + * on-the-fly final merge, in which case we don't have an output tape). + */ +static void +init_tape_buffers(Tuplesortstate *state, int numInputTapes) +{ + int64 availBlocks; + int64 blocksPerTape; + int remainder; + int tapenum; + + /* + * Divide availMem evenly among the number of input tapes. + */ + availBlocks = state->availMem / BLCKSZ; + blocksPerTape = availBlocks / numInputTapes; + remainder = availBlocks % numInputTapes; + USEMEM(state, availBlocks * BLCKSZ); + +#ifdef TRACE_SORT + if (trace_sort) + elog(LOG, "using " INT64_FORMAT " KB of memory for read buffers among %d input tapes", + (availBlocks * BLCKSZ) / 1024, numInputTapes); +#endif + + /* + * Use one page per tape, even if we are out of memory. + * tuplesort_merge_order() should've chosen the number of tapes so that + * this can't happen, but better safe than sorry. (This also protects + * from a negative availMem.) + */ + if (blocksPerTape < 1) + { + blocksPerTape = 1; + remainder = 0; + } + + /* + * Set the buffers for the tapes. + * + * In a multi-phase merge, the tape that is initially used as an output + * tape, will later be rewound and read from, and should also use a large + * buffer at that point. So we must loop up to maxTapes, not just + * numInputTapes! + * + * If there are fewer runs than tapes, we will set the buffer size also + * for tapes that will go completely unused, but that's harmless. + * LogicalTapeAssignReadBufferSize() doesn't allocate the buffer + * immediately, it just sets the size that will be used, when the tape is + * rewound for read, and the tape isn't empty. + */ + for (tapenum = 0; tapenum < state->maxTapes; tapenum++) + { + int64 numBlocks = blocksPerTape + (tapenum < remainder ? 1 : 0); + + LogicalTapeAssignReadBufferSize(state->tapeset, tapenum, + numBlocks * BLCKSZ); + } +} + /* * mergeruns -- merge all the completed initial runs. * @@ -2478,6 +2590,8 @@ mergeruns(Tuplesortstate *state) svTape, svRuns, svDummy; + int numTapes; + int numInputTapes; Assert(state->status == TSS_BUILDRUNS); Assert(state->memtupcount == 0); @@ -2498,6 +2612,56 @@ mergeruns(Tuplesortstate *state) state->sortKeys->abbrev_full_comparator = NULL; } + /* + * Reset tuple memory. We've freed all the tuples that we previously + * allocated. We will use the slab allocator from now on. + */ + MemoryContextDelete(state->tuplecontext); + state->tuplecontext = NULL; + + /* + * We no longer need a large memtuples array. (We will allocate a smaller + * one for the heap later.) + */ + FREEMEM(state, GetMemoryChunkSpace(state->memtuples)); + pfree(state->memtuples); + state->memtuples = NULL; + + /* + * If we had fewer runs than tapes, refund the memory that we imagined we + * would need for the tape buffers of the unused tapes. + * + * numTapes and numInputTapes reflect the actual number of tapes we will + * use. Note that the output tape's tape number is maxTapes - 1, so the + * tape numbers of the used tapes are not consecutive, and you cannot just + * loop from 0 to numTapes to visit all used tapes! + */ + if (state->Level == 1) + { + numInputTapes = state->currentRun; + numTapes = numInputTapes + 1; + FREEMEM(state, (state->maxTapes - numTapes) * TAPE_BUFFER_OVERHEAD); + } + else + { + numInputTapes = state->tapeRange; + numTapes = state->maxTapes; + } + + /* + * Initialize the slab allocator. We need one slab slot per input tape, + * for the tuples in the heap, plus one to hold the tuple last returned + * from tuplesort_gettuple. (If we're sorting pass-by-val Datums, + * however, we don't need to do allocate anything.) + * + * From this point on, we no longer use the USEMEM()/LACKMEM() mechanism + * to track memory usage of individual tuples. + */ + if (state->tuples) + init_slab_allocator(state, numInputTapes + 1); + else + init_slab_allocator(state, 0); + /* * If we produced only one initial run (quite likely if the total data * volume is between 1X and 2X workMem when replacement selection is used, @@ -2514,6 +2678,35 @@ mergeruns(Tuplesortstate *state) return; } + /* + * Use all the spare memory we have available for read buffers for the + * tapes. + * + * We do this only after checking for the case that we produced only one + * initial run, because there is no need to use a large read buffer when + * we're reading from a single tape. With one tape, the I/O pattern will + * be the same regardless of the buffer size. + * + * We don't try to "rebalance" the amount of memory among tapes, when we + * start a new merge phase, even if some tapes can be inactive in the + * phase. That would be hard, because logtape.c doesn't know where one + * run ends and another begins. When a new merge phase begins, and a tape + * doesn't participate in it, its buffer nevertheless already contains + * tuples from the next run on same tape, so we cannot release the buffer. + * That's OK in practice, merge performance isn't that sensitive to the + * amount of buffers used, and most merge phases use all or almost all + * tapes, anyway. + */ + init_tape_buffers(state, numInputTapes); + + /* + * Allocate a new 'memtuples' array, for the heap. It will hold one tuple + * from each input tape. + */ + state->memtupsize = numInputTapes; + state->memtuples = (SortTuple *) palloc(numInputTapes * sizeof(SortTuple)); + USEMEM(state, GetMemoryChunkSpace(state->memtuples)); + /* End of step D2: rewind all output tapes to prepare for merging */ for (tapenum = 0; tapenum < state->tapeRange; tapenum++) LogicalTapeRewind(state->tapeset, tapenum, false); @@ -2544,7 +2737,7 @@ mergeruns(Tuplesortstate *state) /* Tell logtape.c we won't be writing anymore */ LogicalTapeSetForgetFreeSpace(state->tapeset); /* Initialize for the final merge pass */ - beginmerge(state, state->tuples); + beginmerge(state); state->status = TSS_FINALMERGE; return; } @@ -2614,6 +2807,13 @@ mergeruns(Tuplesortstate *state) state->result_tape = state->tp_tapenum[state->tapeRange]; LogicalTapeFreeze(state->tapeset, state->result_tape); state->status = TSS_SORTEDONTAPE; + + /* Release the read buffers of all the other tapes, by rewinding them. */ + for (tapenum = 0; tapenum < state->maxTapes; tapenum++) + { + if (tapenum != state->result_tape) + LogicalTapeRewind(state->tapeset, tapenum, true); + } } /* @@ -2627,16 +2827,12 @@ mergeonerun(Tuplesortstate *state) { int destTape = state->tp_tapenum[state->tapeRange]; int srcTape; - int tupIndex; - SortTuple *tup; - int64 priorAvail, - spaceFreed; /* * Start the merge by loading one tuple from each active source tape into * the heap. We can also decrease the input run/dummy run counts. */ - beginmerge(state, false); + beginmerge(state); /* * Execute merge by repeatedly extracting lowest tuple in heap, writing it @@ -2645,50 +2841,28 @@ mergeonerun(Tuplesortstate *state) */ while (state->memtupcount > 0) { + SortTuple stup; + /* write the tuple to destTape */ - priorAvail = state->availMem; srcTape = state->memtuples[0].tupindex; WRITETUP(state, destTape, &state->memtuples[0]); - /* writetup adjusted total free space, now fix per-tape space */ - spaceFreed = state->availMem - priorAvail; - state->mergeavailmem[srcTape] += spaceFreed; - if ((tupIndex = state->mergenext[srcTape]) == 0) - { - /* out of preloaded data on this tape, try to read more */ - mergepreread(state); - /* if still no data, we've reached end of run on this tape */ - if ((tupIndex = state->mergenext[srcTape]) == 0) - { - /* remove the written-out tuple from the heap */ - tuplesort_heap_delete_top(state, false); - continue; - } - } + + /* recycle the slot of the tuple we just wrote out, for the next read */ + RELEASE_SLAB_SLOT(state, state->memtuples[0].tuple); /* - * pull next preread tuple from list, and replace the written-out - * tuple in the heap with it. + * pull next tuple from the tape, and replace the written-out tuple in + * the heap with it. */ - tup = &state->memtuples[tupIndex]; - state->mergenext[srcTape] = tup->tupindex; - if (state->mergenext[srcTape] == 0) - state->mergelast[srcTape] = 0; - tup->tupindex = srcTape; - tuplesort_heap_replace_top(state, tup, false); - /* put the now-unused memtuples entry on the freelist */ - tup->tupindex = state->mergefreelist; - state->mergefreelist = tupIndex; - state->mergeavailslots[srcTape]++; - } + if (mergereadnext(state, srcTape, &stup)) + { + stup.tupindex = srcTape; + tuplesort_heap_replace_top(state, &stup, false); - /* - * Reset tuple memory. We've freed all of the tuples that we previously - * allocated, but AllocSetFree will have put those chunks of memory on - * particular free lists, bucketed by size class. Thus, although all of - * that memory is free, it is effectively fragmented. Resetting the - * context gets us out from under that problem. - */ - MemoryContextReset(state->tuplecontext); + } + else + tuplesort_heap_delete_top(state, false); + } /* * When the heap empties, we're done. Write an end-of-run marker on the @@ -2711,18 +2885,13 @@ mergeonerun(Tuplesortstate *state) * which tapes contain active input runs in mergeactive[]. Then, load * as many tuples as we can from each active input tape, and finally * fill the merge heap with the first tuple from each active tape. - * - * finalMergeBatch indicates if this is the beginning of a final on-the-fly - * merge where a batched allocation of tuple memory is required. */ static void -beginmerge(Tuplesortstate *state, bool finalMergeBatch) +beginmerge(Tuplesortstate *state) { int activeTapes; int tapenum; int srcTape; - int slotsPerTape; - int64 spacePerTape; /* Heap should be empty here */ Assert(state->memtupcount == 0); @@ -2744,519 +2913,44 @@ beginmerge(Tuplesortstate *state, bool finalMergeBatch) activeTapes++; } } - state->activeTapes = activeTapes; - - /* Clear merge-pass state variables */ - memset(state->mergenext, 0, - state->maxTapes * sizeof(*state->mergenext)); - memset(state->mergelast, 0, - state->maxTapes * sizeof(*state->mergelast)); - state->mergefreelist = 0; /* nothing in the freelist */ - state->mergefirstfree = activeTapes; /* 1st slot avail for preread */ - - if (finalMergeBatch) - { - /* Free outright buffers for tape never actually allocated */ - FREEMEM(state, (state->maxTapes - activeTapes) * TAPE_BUFFER_OVERHEAD); - - /* - * Grow memtuples one last time, since the palloc() overhead no longer - * incurred can make a big difference - */ - batchmemtuples(state); - } - - /* - * Initialize space allocation to let each active input tape have an equal - * share of preread space. - */ Assert(activeTapes > 0); - slotsPerTape = (state->memtupsize - state->mergefirstfree) / activeTapes; - Assert(slotsPerTape > 0); - spacePerTape = MAXALIGN_DOWN(state->availMem / activeTapes); - for (srcTape = 0; srcTape < state->maxTapes; srcTape++) - { - if (state->mergeactive[srcTape]) - { - state->mergeavailslots[srcTape] = slotsPerTape; - state->mergeavailmem[srcTape] = spacePerTape; - } - } - - /* - * Preallocate tuple batch memory for each tape. This is the memory used - * for tuples themselves (not SortTuples), so it's never used by - * pass-by-value datum sorts. Memory allocation is performed here at most - * once per sort, just in advance of the final on-the-fly merge step. - */ - if (finalMergeBatch) - mergebatch(state, spacePerTape); - - /* - * Preread as many tuples as possible (and at least one) from each active - * tape - */ - mergepreread(state); + state->activeTapes = activeTapes; /* Load the merge heap with the first tuple from each input tape */ for (srcTape = 0; srcTape < state->maxTapes; srcTape++) { - int tupIndex = state->mergenext[srcTape]; - SortTuple *tup; + SortTuple tup; - if (tupIndex) + if (mergereadnext(state, srcTape, &tup)) { - tup = &state->memtuples[tupIndex]; - state->mergenext[srcTape] = tup->tupindex; - if (state->mergenext[srcTape] == 0) - state->mergelast[srcTape] = 0; - tup->tupindex = srcTape; - tuplesort_heap_insert(state, tup, false); - /* put the now-unused memtuples entry on the freelist */ - tup->tupindex = state->mergefreelist; - state->mergefreelist = tupIndex; - state->mergeavailslots[srcTape]++; - -#ifdef TRACE_SORT - if (trace_sort && finalMergeBatch) - { - int64 perTapeKB = (spacePerTape + 1023) / 1024; - int64 usedSpaceKB; - int usedSlots; - - /* - * Report how effective batchmemtuples() was in balancing the - * number of slots against the need for memory for the - * underlying tuples (e.g. IndexTuples). The big preread of - * all tapes when switching to FINALMERGE state should be - * fairly representative of memory utilization during the - * final merge step, and in any case is the only point at - * which all tapes are guaranteed to have depleted either - * their batch memory allowance or slot allowance. Ideally, - * both will be completely depleted for every tape by now. - */ - usedSpaceKB = (state->mergecurrent[srcTape] - - state->mergetuples[srcTape] + 1023) / 1024; - usedSlots = slotsPerTape - state->mergeavailslots[srcTape]; - - elog(LOG, "tape %d initially used " INT64_FORMAT " KB of " - INT64_FORMAT " KB batch (%2.3f) and %d out of %d slots " - "(%2.3f)", srcTape, - usedSpaceKB, perTapeKB, - (double) usedSpaceKB / (double) perTapeKB, - usedSlots, slotsPerTape, - (double) usedSlots / (double) slotsPerTape); - } -#endif + tup.tupindex = srcTape; + tuplesort_heap_insert(state, &tup, false); } } } /* - * batchmemtuples - grow memtuples without palloc overhead - * - * When called, availMem should be approximately the amount of memory we'd - * require to allocate memtupsize - memtupcount tuples (not SortTuples/slots) - * that were allocated with palloc() overhead, and in doing so use up all - * allocated slots. However, though slots and tuple memory is in balance - * following the last grow_memtuples() call, that's predicated on the observed - * average tuple size for the "final" grow_memtuples() call, which includes - * palloc overhead. During the final merge pass, where we will arrange to - * squeeze out the palloc overhead, we might need more slots in the memtuples - * array. - * - * To make that happen, arrange for the amount of remaining memory to be - * exactly equal to the palloc overhead multiplied by the current size of - * the memtuples array, force the grow_memtuples flag back to true (it's - * probably but not necessarily false on entry to this routine), and then - * call grow_memtuples. This simulates loading enough tuples to fill the - * whole memtuples array and then having some space left over because of the - * elided palloc overhead. We expect that grow_memtuples() will conclude that - * it can't double the size of the memtuples array but that it can increase - * it by some percentage; but if it does decide to double it, that just means - * that we've never managed to use many slots in the memtuples array, in which - * case doubling it shouldn't hurt anything anyway. - */ -static void -batchmemtuples(Tuplesortstate *state) -{ - int64 refund; - int64 availMemLessRefund; - int memtupsize = state->memtupsize; - - /* Caller error if we have no tapes */ - Assert(state->activeTapes > 0); - - /* For simplicity, assume no memtuples are actually currently counted */ - Assert(state->memtupcount == 0); - - /* - * Refund STANDARDCHUNKHEADERSIZE per tuple. - * - * This sometimes fails to make memory use perfectly balanced, but it - * should never make the situation worse. Note that Assert-enabled builds - * get a larger refund, due to a varying STANDARDCHUNKHEADERSIZE. - */ - refund = memtupsize * STANDARDCHUNKHEADERSIZE; - availMemLessRefund = state->availMem - refund; - - /* - * We need to be sure that we do not cause LACKMEM to become true, else - * the batch allocation size could be calculated as negative, causing - * havoc. Hence, if availMemLessRefund is negative at this point, we must - * do nothing. Moreover, if it's positive but rather small, there's - * little point in proceeding because we could only increase memtuples by - * a small amount, not worth the cost of the repalloc's. We somewhat - * arbitrarily set the threshold at ALLOCSET_DEFAULT_INITSIZE per tape. - * (Note that this does not represent any assumption about tuple sizes.) - */ - if (availMemLessRefund <= - (int64) state->activeTapes * ALLOCSET_DEFAULT_INITSIZE) - return; - - /* - * To establish balanced memory use after refunding palloc overhead, - * temporarily have our accounting indicate that we've allocated all - * memory we're allowed to less that refund, and call grow_memtuples() to - * have it increase the number of slots. - */ - state->growmemtuples = true; - USEMEM(state, availMemLessRefund); - (void) grow_memtuples(state); - state->growmemtuples = false; - /* availMem must stay accurate for spacePerTape calculation */ - FREEMEM(state, availMemLessRefund); - if (LACKMEM(state)) - elog(ERROR, "unexpected out-of-memory situation in tuplesort"); - -#ifdef TRACE_SORT - if (trace_sort) - { - Size OldKb = (memtupsize * sizeof(SortTuple) + 1023) / 1024; - Size NewKb = (state->memtupsize * sizeof(SortTuple) + 1023) / 1024; - - elog(LOG, "grew memtuples %1.2fx from %d (%zu KB) to %d (%zu KB) for final merge", - (double) NewKb / (double) OldKb, - memtupsize, OldKb, - state->memtupsize, NewKb); - } -#endif -} - -/* - * mergebatch - initialize tuple memory in batch - * - * This allows sequential access to sorted tuples buffered in memory from - * tapes/runs on disk during a final on-the-fly merge step. Note that the - * memory is not used for SortTuples, but for the underlying tuples (e.g. - * MinimalTuples). - * - * Note that when batch memory is used, there is a simple division of space - * into large buffers (one per active tape). The conventional incremental - * memory accounting (calling USEMEM() and FREEMEM()) is abandoned. Instead, - * when each tape's memory budget is exceeded, a retail palloc() "overflow" is - * performed, which is then immediately detected in a way that is analogous to - * LACKMEM(). This keeps each tape's use of memory fair, which is always a - * goal. - */ -static void -mergebatch(Tuplesortstate *state, int64 spacePerTape) -{ - int srcTape; - - Assert(state->activeTapes > 0); - Assert(state->tuples); - - /* - * For the purposes of tuplesort's memory accounting, the batch allocation - * is special, and regular memory accounting through USEMEM() calls is - * abandoned (see mergeprereadone()). - */ - for (srcTape = 0; srcTape < state->maxTapes; srcTape++) - { - char *mergetuples; - - if (!state->mergeactive[srcTape]) - continue; - - /* Allocate buffer for each active tape */ - mergetuples = MemoryContextAllocHuge(state->tuplecontext, - spacePerTape); - - /* Initialize state for tape */ - state->mergetuples[srcTape] = mergetuples; - state->mergecurrent[srcTape] = mergetuples; - state->mergetail[srcTape] = mergetuples; - state->mergeoverflow[srcTape] = NULL; - } - - state->batchUsed = true; - state->spacePerTape = spacePerTape; -} - -/* - * mergebatchone - prepare batch memory for one merge input tape - * - * This is called following the exhaustion of preread tuples for one input - * tape. All that actually occurs is that the state for the source tape is - * reset to indicate that all memory may be reused. - * - * This routine must deal with fixing up the tuple that is about to be returned - * to the client, due to "overflow" allocations. - */ -static void -mergebatchone(Tuplesortstate *state, int srcTape, SortTuple *rtup, - bool *should_free) -{ - Assert(state->batchUsed); - - /* - * Tuple about to be returned to caller ("stup") is final preread tuple - * from tape, just removed from the top of the heap. Special steps around - * memory management must be performed for that tuple, to make sure it - * isn't overwritten early. - */ - if (!state->mergeoverflow[srcTape]) - { - Size tupLen; - - /* - * Mark tuple buffer range for reuse, but be careful to move final, - * tail tuple to start of space for next run so that it's available to - * caller when stup is returned, and remains available at least until - * the next tuple is requested. - */ - tupLen = state->mergecurrent[srcTape] - state->mergetail[srcTape]; - state->mergecurrent[srcTape] = state->mergetuples[srcTape]; - MOVETUP(state->mergecurrent[srcTape], state->mergetail[srcTape], - tupLen); - - /* Make SortTuple at top of the merge heap point to new tuple */ - rtup->tuple = (void *) state->mergecurrent[srcTape]; - - state->mergetail[srcTape] = state->mergecurrent[srcTape]; - state->mergecurrent[srcTape] += tupLen; - } - else - { - /* - * Handle an "overflow" retail palloc. - * - * This is needed when we run out of tuple memory for the tape. - */ - state->mergecurrent[srcTape] = state->mergetuples[srcTape]; - state->mergetail[srcTape] = state->mergetuples[srcTape]; - - if (rtup->tuple) - { - Assert(rtup->tuple == (void *) state->mergeoverflow[srcTape]); - /* Caller should free palloc'd tuple */ - *should_free = true; - } - state->mergeoverflow[srcTape] = NULL; - } -} - -/* - * mergebatchfreetape - handle final clean-up for batch memory once tape is - * about to become exhausted - * - * All tuples are returned from tape, but a single final tuple, *rtup, is to be - * passed back to caller. Free tape's batch allocation buffer while ensuring - * that the final tuple is managed appropriately. - */ -static void -mergebatchfreetape(Tuplesortstate *state, int srcTape, SortTuple *rtup, - bool *should_free) -{ - Assert(state->batchUsed); - Assert(state->status == TSS_FINALMERGE); - - /* - * Tuple may or may not already be an overflow allocation from - * mergebatchone() - */ - if (!*should_free && rtup->tuple) - { - /* - * Final tuple still in tape's batch allocation. - * - * Return palloc()'d copy to caller, and have it freed in a similar - * manner to overflow allocation. Otherwise, we'd free batch memory - * and pass back a pointer to garbage. Note that we deliberately - * allocate this in the parent tuplesort context, to be on the safe - * side. - */ - Size tuplen; - void *oldTuple = rtup->tuple; - - tuplen = state->mergecurrent[srcTape] - state->mergetail[srcTape]; - rtup->tuple = MemoryContextAlloc(state->sortcontext, tuplen); - MOVETUP(rtup->tuple, oldTuple, tuplen); - *should_free = true; - } - - /* Free spacePerTape-sized buffer */ - pfree(state->mergetuples[srcTape]); -} - -/* - * mergebatchalloc - allocate memory for one tuple using a batch memory - * "logical allocation". - * - * This is used for the final on-the-fly merge phase only. READTUP() routines - * receive memory from here in place of palloc() and USEMEM() calls. - * - * Tuple tapenum is passed, ensuring each tape's tuples are stored in sorted, - * contiguous order (while allowing safe reuse of memory made available to - * each tape). This maximizes locality of access as tuples are returned by - * final merge. + * mergereadnext - read next tuple from one merge input tape * - * Caller must not subsequently attempt to free memory returned here. In - * general, only mergebatch* functions know about how memory returned from - * here should be freed, and this function's caller must ensure that batch - * memory management code will definitely have the opportunity to do the right - * thing during the final on-the-fly merge. + * Returns false on EOF. */ -static void * -mergebatchalloc(Tuplesortstate *state, int tapenum, Size tuplen) -{ - Size reserve_tuplen = MAXALIGN(tuplen); - char *ret; - - /* Should overflow at most once before mergebatchone() call: */ - Assert(state->mergeoverflow[tapenum] == NULL); - Assert(state->batchUsed); - - /* It should be possible to use precisely spacePerTape memory at once */ - if (state->mergecurrent[tapenum] + reserve_tuplen <= - state->mergetuples[tapenum] + state->spacePerTape) - { - /* - * Usual case -- caller is returned pointer into its tape's buffer, - * and an offset from that point is recorded as where tape has - * consumed up to for current round of preloading. - */ - ret = state->mergetail[tapenum] = state->mergecurrent[tapenum]; - state->mergecurrent[tapenum] += reserve_tuplen; - } - else - { - /* - * Allocate memory, and record as tape's overflow allocation. This - * will be detected quickly, in a similar fashion to a LACKMEM() - * condition, and should not happen again before a new round of - * preloading for caller's tape. Note that we deliberately allocate - * this in the parent tuplesort context, to be on the safe side. - * - * Sometimes, this does not happen because merging runs out of slots - * before running out of memory. - */ - ret = state->mergeoverflow[tapenum] = - MemoryContextAlloc(state->sortcontext, tuplen); - } - - return ret; -} - -/* - * mergepreread - load tuples from merge input tapes - * - * This routine exists to improve sequentiality of reads during a merge pass, - * as explained in the header comments of this file. Load tuples from each - * active source tape until the tape's run is exhausted or it has used up - * its fair share of available memory. In any case, we guarantee that there - * is at least one preread tuple available from each unexhausted input tape. - * - * We invoke this routine at the start of a merge pass for initial load, - * and then whenever any tape's preread data runs out. Note that we load - * as much data as possible from all tapes, not just the one that ran out. - * This is because logtape.c works best with a usage pattern that alternates - * between reading a lot of data and writing a lot of data, so whenever we - * are forced to read, we should fill working memory completely. - * - * In FINALMERGE state, we *don't* use this routine, but instead just preread - * from the single tape that ran dry. There's no read/write alternation in - * that state and so no point in scanning through all the tapes to fix one. - * (Moreover, there may be quite a lot of inactive tapes in that state, since - * we might have had many fewer runs than tapes. In a regular tape-to-tape - * merge we can expect most of the tapes to be active. Plus, only - * FINALMERGE state has to consider memory management for a batch - * allocation.) - */ -static void -mergepreread(Tuplesortstate *state) -{ - int srcTape; - - for (srcTape = 0; srcTape < state->maxTapes; srcTape++) - mergeprereadone(state, srcTape); -} - -/* - * mergeprereadone - load tuples from one merge input tape - * - * Read tuples from the specified tape until it has used up its free memory - * or array slots; but ensure that we have at least one tuple, if any are - * to be had. - */ -static void -mergeprereadone(Tuplesortstate *state, int srcTape) +static bool +mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup) { unsigned int tuplen; - SortTuple stup; - int tupIndex; - int64 priorAvail, - spaceUsed; if (!state->mergeactive[srcTape]) - return; /* tape's run is already exhausted */ - - /* - * Manage per-tape availMem. Only actually matters when batch memory not - * in use. - */ - priorAvail = state->availMem; - state->availMem = state->mergeavailmem[srcTape]; + return false; /* tape's run is already exhausted */ - /* - * When batch memory is used if final on-the-fly merge, only mergeoverflow - * test is relevant; otherwise, only LACKMEM() test is relevant. - */ - while ((state->mergeavailslots[srcTape] > 0 && - state->mergeoverflow[srcTape] == NULL && !LACKMEM(state)) || - state->mergenext[srcTape] == 0) + /* read next tuple, if any */ + if ((tuplen = getlen(state, srcTape, true)) == 0) { - /* read next tuple, if any */ - if ((tuplen = getlen(state, srcTape, true)) == 0) - { - state->mergeactive[srcTape] = false; - break; - } - READTUP(state, &stup, srcTape, tuplen); - /* find a free slot in memtuples[] for it */ - tupIndex = state->mergefreelist; - if (tupIndex) - state->mergefreelist = state->memtuples[tupIndex].tupindex; - else - { - tupIndex = state->mergefirstfree++; - Assert(tupIndex < state->memtupsize); - } - state->mergeavailslots[srcTape]--; - /* store tuple, append to list for its tape */ - stup.tupindex = 0; - state->memtuples[tupIndex] = stup; - if (state->mergelast[srcTape]) - state->memtuples[state->mergelast[srcTape]].tupindex = tupIndex; - else - state->mergenext[srcTape] = tupIndex; - state->mergelast[srcTape] = tupIndex; + state->mergeactive[srcTape] = false; + return false; } - /* update per-tape and global availmem counts */ - spaceUsed = state->mergeavailmem[srcTape] - state->availMem; - state->mergeavailmem[srcTape] = state->availMem; - state->availMem = priorAvail - spaceUsed; + READTUP(state, stup, srcTape, tuplen); + + return true; } /* @@ -3441,9 +3135,9 @@ dumpbatch(Tuplesortstate *state, bool alltuples) /* * Reset tuple memory. We've freed all of the tuples that we previously * allocated. It's important to avoid fragmentation when there is a stark - * change in allocation patterns due to the use of batch memory. - * Fragmentation due to AllocSetFree's bucketing by size class might be - * particularly bad if this step wasn't taken. + * change in the sizes of incoming tuples. Fragmentation due to + * AllocSetFree's bucketing by size class might be particularly bad if + * this step wasn't taken. */ MemoryContextReset(state->tuplecontext); @@ -3901,38 +3595,31 @@ markrunend(Tuplesortstate *state, int tapenum) } /* - * Get memory for tuple from within READTUP() routine. Allocate - * memory and account for that, or consume from tape's batch - * allocation. + * Get memory for tuple from within READTUP() routine. * - * Memory returned here in the final on-the-fly merge case is recycled - * from tape's batch allocation. Otherwise, callers must pfree() or - * reset tuple child memory context, and account for that with a - * FREEMEM(). Currently, this only ever needs to happen in WRITETUP() - * routines. + * We use next free slot from the slab allocator, or palloc() if the tuple + * is too large for that. */ static void * -readtup_alloc(Tuplesortstate *state, int tapenum, Size tuplen) +readtup_alloc(Tuplesortstate *state, Size tuplen) { - if (state->batchUsed) - { - /* - * No USEMEM() call, because during final on-the-fly merge accounting - * is based on tape-private state. ("Overflow" allocations are - * detected as an indication that a new round or preloading is - * required. Preloading marks existing contents of tape's batch buffer - * for reuse.) - */ - return mergebatchalloc(state, tapenum, tuplen); - } + SlabSlot *buf; + + /* + * We pre-allocate enough slots in the slab arena that we should never run + * out. + */ + Assert(state->slabFreeHead); + + if (tuplen > SLAB_SLOT_SIZE || !state->slabFreeHead) + return MemoryContextAlloc(state->sortcontext, tuplen); else { - char *ret; + buf = state->slabFreeHead; + /* Reuse this slot */ + state->slabFreeHead = buf->nextfree; - /* Batch allocation yet to be performed */ - ret = MemoryContextAlloc(state->tuplecontext, tuplen); - USEMEM(state, GetMemoryChunkSpace(ret)); - return ret; + return buf; } } @@ -4101,8 +3788,11 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup) LogicalTapeWrite(state->tapeset, tapenum, (void *) &tuplen, sizeof(tuplen)); - FREEMEM(state, GetMemoryChunkSpace(tuple)); - heap_free_minimal_tuple(tuple); + if (!state->slabAllocatorUsed) + { + FREEMEM(state, GetMemoryChunkSpace(tuple)); + heap_free_minimal_tuple(tuple); + } } static void @@ -4111,7 +3801,7 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup, { unsigned int tupbodylen = len - sizeof(int); unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET; - MinimalTuple tuple = (MinimalTuple) readtup_alloc(state, tapenum, tuplen); + MinimalTuple tuple = (MinimalTuple) readtup_alloc(state, tuplen); char *tupbody = (char *) tuple + MINIMAL_TUPLE_DATA_OFFSET; HeapTupleData htup; @@ -4132,12 +3822,6 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup, &stup->isnull1); } -static void -movetup_heap(void *dest, void *src, unsigned int len) -{ - memmove(dest, src, len); -} - /* * Routines specialized for the CLUSTER case (HeapTuple data, with * comparisons per a btree index definition) @@ -4344,8 +4028,11 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup) LogicalTapeWrite(state->tapeset, tapenum, &tuplen, sizeof(tuplen)); - FREEMEM(state, GetMemoryChunkSpace(tuple)); - heap_freetuple(tuple); + if (!state->slabAllocatorUsed) + { + FREEMEM(state, GetMemoryChunkSpace(tuple)); + heap_freetuple(tuple); + } } static void @@ -4354,7 +4041,6 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup, { unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int); HeapTuple tuple = (HeapTuple) readtup_alloc(state, - tapenum, t_len + HEAPTUPLESIZE); /* Reconstruct the HeapTupleData header */ @@ -4379,19 +4065,6 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup, &stup->isnull1); } -static void -movetup_cluster(void *dest, void *src, unsigned int len) -{ - HeapTuple tuple; - - memmove(dest, src, len); - - /* Repoint the HeapTupleData header */ - tuple = (HeapTuple) dest; - tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE); -} - - /* * Routines specialized for IndexTuple case * @@ -4659,8 +4332,11 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup) LogicalTapeWrite(state->tapeset, tapenum, (void *) &tuplen, sizeof(tuplen)); - FREEMEM(state, GetMemoryChunkSpace(tuple)); - pfree(tuple); + if (!state->slabAllocatorUsed) + { + FREEMEM(state, GetMemoryChunkSpace(tuple)); + pfree(tuple); + } } static void @@ -4668,7 +4344,7 @@ readtup_index(Tuplesortstate *state, SortTuple *stup, int tapenum, unsigned int len) { unsigned int tuplen = len - sizeof(unsigned int); - IndexTuple tuple = (IndexTuple) readtup_alloc(state, tapenum, tuplen); + IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen); LogicalTapeReadExact(state->tapeset, tapenum, tuple, tuplen); @@ -4683,12 +4359,6 @@ readtup_index(Tuplesortstate *state, SortTuple *stup, &stup->isnull1); } -static void -movetup_index(void *dest, void *src, unsigned int len) -{ - memmove(dest, src, len); -} - /* * Routines specialized for DatumTuple case */ @@ -4755,7 +4425,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup) LogicalTapeWrite(state->tapeset, tapenum, (void *) &writtenlen, sizeof(writtenlen)); - if (stup->tuple) + if (!state->slabAllocatorUsed && stup->tuple) { FREEMEM(state, GetMemoryChunkSpace(stup->tuple)); pfree(stup->tuple); @@ -4785,7 +4455,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup, } else { - void *raddr = readtup_alloc(state, tapenum, tuplen); + void *raddr = readtup_alloc(state, tuplen); LogicalTapeReadExact(state->tapeset, tapenum, raddr, tuplen); @@ -4799,12 +4469,6 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup, &tuplen, sizeof(tuplen)); } -static void -movetup_datum(void *dest, void *src, unsigned int len) -{ - memmove(dest, src, len); -} - /* * Convenience routine to free a tuple previously loaded into sort memory */ diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h index fa1e992082..362a6196dc 100644 --- a/src/include/utils/logtape.h +++ b/src/include/utils/logtape.h @@ -39,6 +39,8 @@ extern bool LogicalTapeSeek(LogicalTapeSet *lts, int tapenum, long blocknum, int offset); extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum, long *blocknum, int *offset); +extern void LogicalTapeAssignReadBufferSize(LogicalTapeSet *lts, int tapenum, + size_t bufsize); extern long LogicalTapeSetBlocks(LogicalTapeSet *lts); #endif /* LOGTAPE_H */ From 83c2492002162bf79d2a0811bff5724e395909d7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 3 Oct 2016 10:07:39 -0400 Subject: [PATCH 263/871] Enforce a specific order for probing library loadability in pg_upgrade. pg_upgrade checks whether all the shared libraries used in the old cluster are also available in the new one by issuing LOAD for each library name. Previously, it cared not what order it did the LOADs in. Ideally it should not have to care, but currently the transform modules in contrib fail unless both the language and datatype modules they depend on are loaded first. A backend-side solution for that looks possible but probably not back-patchable, so as a stopgap measure, let's do the LOAD tests in order by library name length. That should fix the problem for reasonably-named transform modules, eg "hstore_plpython" will be loaded after both "hstore" and "plpython". (Yeah, it's a hack.) In a larger sense, having a predictable order of these probes is a good thing, since it will make upgrades predictably work or not work in the face of inter-library dependencies. Also, this patch replaces O(N^2) de-duplication logic with O(N log N) logic, which could matter in installations with very many databases. So I don't foresee reverting this even after we have a proper fix for the library-dependency problem. In passing, improve a couple of SQL queries used here. Per complaint from Andrew Dunstan that pg_upgrade'ing the transform contrib modules failed. Back-patch to 9.5 where transform modules were introduced. Discussion: --- src/bin/pg_upgrade/function.c | 97 ++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 30 deletions(-) diff --git a/src/bin/pg_upgrade/function.c b/src/bin/pg_upgrade/function.c index b432b544b2..30093407da 100644 --- a/src/bin/pg_upgrade/function.c +++ b/src/bin/pg_upgrade/function.c @@ -12,6 +12,31 @@ #include "pg_upgrade.h" #include "access/transam.h" +#include "catalog/pg_language.h" + + +/* + * qsort comparator for pointers to library names + * + * We sort first by name length, then alphabetically for names of the same + * length. This is to ensure that, eg, "hstore_plpython" sorts after both + * "hstore" and "plpython"; otherwise transform modules will probably fail + * their LOAD tests. (The backend ought to cope with that consideration, + * but it doesn't yet, and even when it does it'll still be a good idea + * to have a predictable order of probing here.) + */ +static int +library_name_compare(const void *p1, const void *p2) +{ + const char *str1 = *(const char *const *) p1; + const char *str2 = *(const char *const *) p2; + int slen1 = strlen(str1); + int slen2 = strlen(str2); + + if (slen1 != slen2) + return slen1 - slen2; + return strcmp(str1, str2); +} /* @@ -38,17 +63,15 @@ get_loadable_libraries(void) PGconn *conn = connectToServer(&old_cluster, active_db->db_name); /* - * Fetch all libraries referenced in this DB. We can't exclude the - * "pg_catalog" schema because, while such functions are not - * explicitly dumped by pg_dump, they do reference implicit objects - * that pg_dump does dump, e.g. CREATE LANGUAGE plperl. + * Fetch all libraries containing non-built-in C functions in this DB. */ ress[dbnum] = executeQueryOrDie(conn, "SELECT DISTINCT probin " - "FROM pg_catalog.pg_proc " - "WHERE prolang = 13 /* C */ AND " + "FROM pg_catalog.pg_proc " + "WHERE prolang = %u AND " "probin IS NOT NULL AND " "oid >= %u;", + ClanguageId, FirstNormalObjectId); totaltups += PQntuples(ress[dbnum]); @@ -69,13 +92,15 @@ get_loadable_libraries(void) res = executeQueryOrDie(conn, "SELECT 1 " - "FROM pg_catalog.pg_proc JOIN pg_namespace " - " ON pronamespace = pg_namespace.oid " + "FROM pg_catalog.pg_proc p " + " JOIN pg_catalog.pg_namespace n " + " ON pronamespace = n.oid " "WHERE proname = 'plpython_call_handler' AND " "nspname = 'public' AND " - "prolang = 13 /* C */ AND " + "prolang = %u AND " "probin = '$libdir/plpython' AND " - "pg_proc.oid >= %u;", + "p.oid >= %u;", + ClanguageId, FirstNormalObjectId); if (PQntuples(res) > 0) { @@ -112,13 +137,18 @@ get_loadable_libraries(void) if (found_public_plpython_handler) pg_fatal("Remove the problem functions from the old cluster to continue.\n"); - /* Allocate what's certainly enough space */ - os_info.libraries = (char **) pg_malloc(totaltups * sizeof(char *)); - /* - * Now remove duplicates across DBs. This is pretty inefficient code, but - * there probably aren't enough entries to matter. + * Now we want to remove duplicates across DBs and sort the library names + * into order. This avoids multiple probes of the same library, and + * ensures that libraries are probed in a consistent order, which is + * important for reproducible behavior if one library depends on another. + * + * First transfer all the names into one array, then sort, then remove + * duplicates. Note: we strdup each name in the first loop so that we can + * safely clear the PGresults in the same loop. This is a bit wasteful + * but it's unlikely there are enough names to matter. */ + os_info.libraries = (char **) pg_malloc(totaltups * sizeof(char *)); totaltups = 0; for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++) @@ -131,27 +161,34 @@ get_loadable_libraries(void) for (rowno = 0; rowno < ntups; rowno++) { char *lib = PQgetvalue(res, rowno, 0); - bool dup = false; - int n; - for (n = 0; n < totaltups; n++) - { - if (strcmp(lib, os_info.libraries[n]) == 0) - { - dup = true; - break; - } - } - if (!dup) - os_info.libraries[totaltups++] = pg_strdup(lib); + os_info.libraries[totaltups++] = pg_strdup(lib); } - PQclear(res); } - os_info.num_libraries = totaltups; - pg_free(ress); + + if (totaltups > 1) + { + int i, + lastnondup; + + qsort((void *) os_info.libraries, totaltups, sizeof(char *), + library_name_compare); + + for (i = 1, lastnondup = 0; i < totaltups; i++) + { + if (strcmp(os_info.libraries[i], + os_info.libraries[lastnondup]) != 0) + os_info.libraries[++lastnondup] = os_info.libraries[i]; + else + pg_free(os_info.libraries[i]); + } + totaltups = lastnondup + 1; + } + + os_info.num_libraries = totaltups; } From 814b9e9b8edf36cac65e0d8fcef17e50a04b1617 Mon Sep 17 00:00:00 2001 From: Stephen Frost Date: Mon, 3 Oct 2016 16:22:57 -0400 Subject: [PATCH 264/871] Fix RLS with COPY (col1, col2) FROM tab Attempting to COPY a subset of columns from a table with RLS enabled would fail due to an invalid query being constructed (using a single ColumnRef with the list of fields to exact in 'fields', but that's for the different levels of an indirection for a single column, not for specifying multiple columns). Correct by building a ColumnRef and then RestTarget for each column being requested and then adding those to the targetList for the select query. Include regression tests to hopefully catch if this is broken again in the future. Patch-By: Adam Brightwell Reviewed-By: Michael Paquier --- src/backend/commands/copy.c | 63 ++++++++++++++++++----- src/test/regress/expected/copy2.out | 78 +++++++++++++++++++++++++++++ src/test/regress/sql/copy2.sql | 63 +++++++++++++++++++++++ 3 files changed, 192 insertions(+), 12 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 432b0ca67b..457c9bbd74 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -871,6 +871,7 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, uint64 *processed) ColumnRef *cr; ResTarget *target; RangeVar *from; + List *targetList = NIL; if (is_from) ereport(ERROR, @@ -878,21 +879,59 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, uint64 *processed) errmsg("COPY FROM not supported with row-level security"), errhint("Use INSERT statements instead."))); - /* Build target list */ - cr = makeNode(ColumnRef); - + /* + * Build target list + * + * If no columns are specified in the attribute list of the COPY + * command, then the target list is 'all' columns. Therefore, '*' + * should be used as the target list for the resulting SELECT + * statement. + * + * In the case that columns are specified in the attribute list, + * create a ColumnRef and ResTarget for each column and add them to + * the target list for the resulting SELECT statement. + */ if (!stmt->attlist) + { + cr = makeNode(ColumnRef); cr->fields = list_make1(makeNode(A_Star)); - else - cr->fields = stmt->attlist; + cr->location = -1; + + target = makeNode(ResTarget); + target->name = NULL; + target->indirection = NIL; + target->val = (Node *) cr; + target->location = -1; - cr->location = 1; + targetList = list_make1(target); + } + else + { + ListCell *lc; - target = makeNode(ResTarget); - target->name = NULL; - target->indirection = NIL; - target->val = (Node *) cr; - target->location = 1; + foreach(lc, stmt->attlist) + { + /* + * Build the ColumnRef for each column. The ColumnRef + * 'fields' property is a String 'Value' node (see + * nodes/value.h) that corresponds to the column name + * respectively. + */ + cr = makeNode(ColumnRef); + cr->fields = list_make1(lfirst(lc)); + cr->location = -1; + + /* Build the ResTarget and add the ColumnRef to it. */ + target = makeNode(ResTarget); + target->name = NULL; + target->indirection = NIL; + target->val = (Node *) cr; + target->location = -1; + + /* Add each column to the SELECT statement's target list */ + targetList = lappend(targetList, target); + } + } /* * Build RangeVar for from clause, fully qualified based on the @@ -903,7 +942,7 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, uint64 *processed) /* Build query */ select = makeNode(SelectStmt); - select->targetList = list_make1(target); + select->targetList = targetList; select->fromClause = list_make1(from); query = (Node *) select; diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 5f6260a8f1..b3c215cf28 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -460,9 +460,87 @@ select * from check_con_tbl; (2 rows) +-- test with RLS enabled. +CREATE ROLE regress_rls_copy_user; +CREATE ROLE regress_rls_copy_user_colperms; +CREATE TABLE rls_t1 (a int, b int, c int); +COPY rls_t1 (a, b, c) from stdin; +CREATE POLICY p1 ON rls_t1 FOR SELECT USING (a % 2 = 0); +ALTER TABLE rls_t1 ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_t1 FORCE ROW LEVEL SECURITY; +GRANT SELECT ON TABLE rls_t1 TO regress_rls_copy_user; +GRANT SELECT (a, b) ON TABLE rls_t1 TO regress_rls_copy_user_colperms; +-- all columns +COPY rls_t1 TO stdout; +1 4 1 +2 3 2 +3 2 3 +4 1 4 +COPY rls_t1 (a, b, c) TO stdout; +1 4 1 +2 3 2 +3 2 3 +4 1 4 +-- subset of columns +COPY rls_t1 (a) TO stdout; +1 +2 +3 +4 +COPY rls_t1 (a, b) TO stdout; +1 4 +2 3 +3 2 +4 1 +-- column reordering +COPY rls_t1 (b, a) TO stdout; +4 1 +3 2 +2 3 +1 4 +SET SESSION AUTHORIZATION regress_rls_copy_user; +-- all columns +COPY rls_t1 TO stdout; +2 3 2 +4 1 4 +COPY rls_t1 (a, b, c) TO stdout; +2 3 2 +4 1 4 +-- subset of columns +COPY rls_t1 (a) TO stdout; +2 +4 +COPY rls_t1 (a, b) TO stdout; +2 3 +4 1 +-- column reordering +COPY rls_t1 (b, a) TO stdout; +3 2 +1 4 +RESET SESSION AUTHORIZATION; +SET SESSION AUTHORIZATION regress_rls_copy_user_colperms; +-- attempt all columns (should fail) +COPY rls_t1 TO stdout; +ERROR: permission denied for relation rls_t1 +COPY rls_t1 (a, b, c) TO stdout; +ERROR: permission denied for relation rls_t1 +-- try to copy column with no privileges (should fail) +COPY rls_t1 (c) TO stdout; +ERROR: permission denied for relation rls_t1 +-- subset of columns (should succeed) +COPY rls_t1 (a) TO stdout; +2 +4 +COPY rls_t1 (a, b) TO stdout; +2 3 +4 1 +RESET SESSION AUTHORIZATION; DROP TABLE forcetest; DROP TABLE vistest; DROP FUNCTION truncate_in_subxact(); DROP TABLE x, y; +DROP TABLE rls_t1 CASCADE; +DROP ROLE regress_rls_copy_user; +DROP ROLE regress_rls_copy_user_colperms; DROP FUNCTION fn_x_before(); DROP FUNCTION fn_x_after(); diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index 39a9deb8f7..89d0a39eb9 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -327,9 +327,72 @@ copy check_con_tbl from stdin; \. select * from check_con_tbl; +-- test with RLS enabled. +CREATE ROLE regress_rls_copy_user; +CREATE ROLE regress_rls_copy_user_colperms; +CREATE TABLE rls_t1 (a int, b int, c int); + +COPY rls_t1 (a, b, c) from stdin; +1 4 1 +2 3 2 +3 2 3 +4 1 4 +\. + +CREATE POLICY p1 ON rls_t1 FOR SELECT USING (a % 2 = 0); +ALTER TABLE rls_t1 ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_t1 FORCE ROW LEVEL SECURITY; + +GRANT SELECT ON TABLE rls_t1 TO regress_rls_copy_user; +GRANT SELECT (a, b) ON TABLE rls_t1 TO regress_rls_copy_user_colperms; + +-- all columns +COPY rls_t1 TO stdout; +COPY rls_t1 (a, b, c) TO stdout; + +-- subset of columns +COPY rls_t1 (a) TO stdout; +COPY rls_t1 (a, b) TO stdout; + +-- column reordering +COPY rls_t1 (b, a) TO stdout; + +SET SESSION AUTHORIZATION regress_rls_copy_user; + +-- all columns +COPY rls_t1 TO stdout; +COPY rls_t1 (a, b, c) TO stdout; + +-- subset of columns +COPY rls_t1 (a) TO stdout; +COPY rls_t1 (a, b) TO stdout; + +-- column reordering +COPY rls_t1 (b, a) TO stdout; + +RESET SESSION AUTHORIZATION; + +SET SESSION AUTHORIZATION regress_rls_copy_user_colperms; + +-- attempt all columns (should fail) +COPY rls_t1 TO stdout; +COPY rls_t1 (a, b, c) TO stdout; + +-- try to copy column with no privileges (should fail) +COPY rls_t1 (c) TO stdout; + +-- subset of columns (should succeed) +COPY rls_t1 (a) TO stdout; +COPY rls_t1 (a, b) TO stdout; + +RESET SESSION AUTHORIZATION; + DROP TABLE forcetest; DROP TABLE vistest; DROP FUNCTION truncate_in_subxact(); DROP TABLE x, y; +DROP TABLE rls_t1 CASCADE; +DROP ROLE regress_rls_copy_user; +DROP ROLE regress_rls_copy_user_colperms; DROP FUNCTION fn_x_before(); DROP FUNCTION fn_x_after(); From 6bc811c992a804bdb8d228ce0be9f0a8e7198df6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 3 Oct 2016 16:40:05 -0400 Subject: [PATCH 265/871] Show a sensible value in pg_settings.unit for GUC_UNIT_XSEGS variables. Commit 88e982302 invented GUC_UNIT_XSEGS for min_wal_size and max_wal_size, but neglected to make it display sensibly in pg_settings.unit (by adding a case to the switch in GetConfigOptionByNum). Fix that, and adjust said switch to throw a run-time error the next time somebody forgets. In passing, avoid using a static buffer for the output string --- the rest of this function pstrdup's from a local buffer, and I see no very good reason why the units code should do it differently and less safely. Per report from Otar Shavadze. Back-patch to 9.5 where the new unit type was added. Report: --- src/backend/utils/misc/guc.c | 20 ++++++++++++++------ src/include/utils/guc.h | 4 ++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index cced814d6a..622279b058 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -8016,20 +8016,23 @@ GetConfigOptionByNum(int varnum, const char **values, bool *noshow) /* unit */ if (conf->vartype == PGC_INT) { - static char buf[8]; - switch (conf->flags & (GUC_UNIT_MEMORY | GUC_UNIT_TIME)) { case GUC_UNIT_KB: values[2] = "kB"; break; case GUC_UNIT_BLOCKS: - snprintf(buf, sizeof(buf), "%dkB", BLCKSZ / 1024); - values[2] = buf; + snprintf(buffer, sizeof(buffer), "%dkB", BLCKSZ / 1024); + values[2] = pstrdup(buffer); break; case GUC_UNIT_XBLOCKS: - snprintf(buf, sizeof(buf), "%dkB", XLOG_BLCKSZ / 1024); - values[2] = buf; + snprintf(buffer, sizeof(buffer), "%dkB", XLOG_BLCKSZ / 1024); + values[2] = pstrdup(buffer); + break; + case GUC_UNIT_XSEGS: + snprintf(buffer, sizeof(buffer), "%dMB", + XLOG_SEG_SIZE / (1024 * 1024)); + values[2] = pstrdup(buffer); break; case GUC_UNIT_MS: values[2] = "ms"; @@ -8040,7 +8043,12 @@ GetConfigOptionByNum(int varnum, const char **values, bool *noshow) case GUC_UNIT_MIN: values[2] = "min"; break; + case 0: + values[2] = NULL; + break; default: + elog(ERROR, "unrecognized GUC units value: %d", + conf->flags & (GUC_UNIT_MEMORY | GUC_UNIT_TIME)); values[2] = NULL; break; } diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index e1de1a5d06..0bf9f21067 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -219,12 +219,12 @@ typedef enum #define GUC_UNIT_BLOCKS 0x2000 /* value is in blocks */ #define GUC_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ #define GUC_UNIT_XSEGS 0x4000 /* value is in xlog segments */ -#define GUC_UNIT_MEMORY 0xF000 /* mask for KB, BLOCKS, XBLOCKS */ +#define GUC_UNIT_MEMORY 0xF000 /* mask for size-related units */ #define GUC_UNIT_MS 0x10000 /* value is in milliseconds */ #define GUC_UNIT_S 0x20000 /* value is in seconds */ #define GUC_UNIT_MIN 0x30000 /* value is in minutes */ -#define GUC_UNIT_TIME 0xF0000 /* mask for MS, S, MIN */ +#define GUC_UNIT_TIME 0xF0000 /* mask for time-related units */ #define GUC_UNIT (GUC_UNIT_MEMORY | GUC_UNIT_TIME) From d51924be886c2a05e691fa05b16cb6b30ab8370f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 3 Oct 2016 22:27:11 -0400 Subject: [PATCH 266/871] Convert contrib/hstore_plpython to not use direct linking to other modules. Previously, on most platforms, we allowed hstore_plpython's references to hstore and plpython to be unresolved symbols at link time, trusting the dynamic linker to resolve them when the module is loaded. This has a number of problems, the worst being that the dynamic linker does not know where the references come from and can do nothing but fail if those other modules haven't been loaded. We've more or less gotten away with that for the limited use-case of datatype transform modules, but even there, it requires some awkward hacks, most recently commit 83c249200. Instead, let's not treat these references as linker-resolvable at all, but use function pointers that are manually filled in by the module's _PG_init function. There are few enough contact points that this doesn't seem unmaintainable, at least for these use-cases. (Note that the same technique wouldn't work at all for decoupling from libpython itself, but fortunately that's just a standard shared library and can be linked to normally.) This is an initial patch that just converts hstore_plpython. If the buildfarm doesn't find any fatal problems, I'll work on the other transform modules soon. Tom Lane, per an idea of Andres Freund's. Discussion: <2652.1475512158@sss.pgh.pa.us> --- contrib/hstore_plpython/Makefile | 19 +++--- contrib/hstore_plpython/hstore_plpython.c | 65 +++++++++++++++++++ .../hstore_plpython2u--1.0.sql | 5 -- .../hstore_plpython3u--1.0.sql | 5 -- .../hstore_plpython/hstore_plpythonu--1.0.sql | 5 -- src/tools/msvc/Mkvcbuild.pm | 3 +- 6 files changed, 76 insertions(+), 26 deletions(-) diff --git a/contrib/hstore_plpython/Makefile b/contrib/hstore_plpython/Makefile index c4dad6f111..a55c9a162c 100644 --- a/contrib/hstore_plpython/Makefile +++ b/contrib/hstore_plpython/Makefile @@ -4,7 +4,7 @@ MODULE_big = hstore_plpython$(python_majorversion) OBJS = hstore_plpython.o $(WIN32RES) PGFILEDESC = "hstore_plpython - hstore transform for plpython" -PG_CPPFLAGS = -I$(top_srcdir)/src/pl/plpython $(python_includespec) -I$(top_srcdir)/contrib/hstore +PG_CPPFLAGS = -I$(top_srcdir)/src/pl/plpython $(python_includespec) -I$(top_srcdir)/contrib/hstore -DPLPYTHON_LIBNAME='"plpython$(python_majorversion)"' EXTENSION = hstore_plpythonu hstore_plpython2u hstore_plpython3u DATA = hstore_plpythonu--1.0.sql hstore_plpython2u--1.0.sql hstore_plpython3u--1.0.sql @@ -23,19 +23,18 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -# In configurations that forbid undefined symbols in libraries, link with each -# dependency. This does preclude pgxs builds. +# We must link libpython explicitly ifeq ($(PORTNAME), aix) rpathdir = $(pkglibdir):$(python_libdir) -SHLIB_LINK += ../hstore/libhstore.exp $(python_libspec) $(python_additional_libs) $(sort $(wildcard ../../src/pl/plpython/libplpython*.exp)) -endif +SHLIB_LINK += $(python_libspec) $(python_additional_libs) +else ifeq ($(PORTNAME), win32) -SHLIB_LINK += ../hstore/libhstore.a $(sort $(wildcard ../../src/pl/plpython/libpython*.a)) $(sort $(wildcard ../../src/pl/plpython/libplpython*.a)) +# ... see silliness in plpython Makefile ... +SHLIB_LINK += $(sort $(wildcard ../../src/pl/plpython/libpython*.a)) +else +rpathdir = $(python_libdir) +SHLIB_LINK += $(python_libspec) endif - -ifeq ($(PORTNAME), cygwin) -SHLIB_LINK += -L../hstore -lhstore -L../../src/pl/plpython \ - -lplpython$(python_majorversion) $(python_libspec) endif REGRESS_OPTS += --load-extension=hstore diff --git a/contrib/hstore_plpython/hstore_plpython.c b/contrib/hstore_plpython/hstore_plpython.c index 6f2751a8df..ce296ca486 100644 --- a/contrib/hstore_plpython/hstore_plpython.c +++ b/contrib/hstore_plpython/hstore_plpython.c @@ -1,4 +1,5 @@ #include "postgres.h" + #include "fmgr.h" #include "plpython.h" #include "plpy_typeio.h" @@ -6,6 +7,70 @@ PG_MODULE_MAGIC; +extern void _PG_init(void); + +/* Linkage to functions in plpython module */ +typedef char *(*PLyObject_AsString_t) (PyObject *plrv); + +static PLyObject_AsString_t PLyObject_AsString_p; + +/* Linkage to functions in hstore module */ +typedef HStore *(*hstoreUpgrade_t) (Datum orig); +typedef int (*hstoreUniquePairs_t) (Pairs *a, int32 l, int32 *buflen); +typedef HStore *(*hstorePairs_t) (Pairs *pairs, int32 pcount, int32 buflen); +typedef size_t (*hstoreCheckKeyLen_t) (size_t len); +typedef size_t (*hstoreCheckValLen_t) (size_t len); + +static hstoreUpgrade_t hstoreUpgrade_p; +static hstoreUniquePairs_t hstoreUniquePairs_p; +static hstorePairs_t hstorePairs_p; +static hstoreCheckKeyLen_t hstoreCheckKeyLen_p; +static hstoreCheckValLen_t hstoreCheckValLen_p; + + +/* + * Module initialize function: fetch function pointers for cross-module calls. + */ +void +_PG_init(void) +{ + /* These asserts verify that typedefs above match original declarations */ + AssertVariableIsOfType(&PLyObject_AsString, PLyObject_AsString_t); + AssertVariableIsOfType(&hstoreUpgrade, hstoreUpgrade_t); + AssertVariableIsOfType(&hstoreUniquePairs, hstoreUniquePairs_t); + AssertVariableIsOfType(&hstorePairs, hstorePairs_t); + AssertVariableIsOfType(&hstoreCheckKeyLen, hstoreCheckKeyLen_t); + AssertVariableIsOfType(&hstoreCheckValLen, hstoreCheckValLen_t); + + PLyObject_AsString_p = (PLyObject_AsString_t) + load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLyObject_AsString", + true, NULL); + hstoreUpgrade_p = (hstoreUpgrade_t) + load_external_function("$libdir/hstore", "hstoreUpgrade", + true, NULL); + hstoreUniquePairs_p = (hstoreUniquePairs_t) + load_external_function("$libdir/hstore", "hstoreUniquePairs", + true, NULL); + hstorePairs_p = (hstorePairs_t) + load_external_function("$libdir/hstore", "hstorePairs", + true, NULL); + hstoreCheckKeyLen_p = (hstoreCheckKeyLen_t) + load_external_function("$libdir/hstore", "hstoreCheckKeyLen", + true, NULL); + hstoreCheckValLen_p = (hstoreCheckValLen_t) + load_external_function("$libdir/hstore", "hstoreCheckValLen", + true, NULL); +} + + +/* These defines must be after the module init function */ +#define PLyObject_AsString PLyObject_AsString_p +#define hstoreUpgrade hstoreUpgrade_p +#define hstoreUniquePairs hstoreUniquePairs_p +#define hstorePairs hstorePairs_p +#define hstoreCheckKeyLen hstoreCheckKeyLen_p +#define hstoreCheckValLen hstoreCheckValLen_p + PG_FUNCTION_INFO_V1(hstore_to_plpython); diff --git a/contrib/hstore_plpython/hstore_plpython2u--1.0.sql b/contrib/hstore_plpython/hstore_plpython2u--1.0.sql index e3aea6399e..800765f3f0 100644 --- a/contrib/hstore_plpython/hstore_plpython2u--1.0.sql +++ b/contrib/hstore_plpython/hstore_plpython2u--1.0.sql @@ -3,11 +3,6 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION hstore_plpython2u" to load this file. \quit --- make sure the prerequisite libraries are loaded -LOAD 'plpython2'; -SELECT NULL::hstore; - - CREATE FUNCTION hstore_to_plpython2(val internal) RETURNS internal LANGUAGE C STRICT IMMUTABLE AS 'MODULE_PATHNAME', 'hstore_to_plpython'; diff --git a/contrib/hstore_plpython/hstore_plpython3u--1.0.sql b/contrib/hstore_plpython/hstore_plpython3u--1.0.sql index a964a49059..0b410ab183 100644 --- a/contrib/hstore_plpython/hstore_plpython3u--1.0.sql +++ b/contrib/hstore_plpython/hstore_plpython3u--1.0.sql @@ -3,11 +3,6 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION hstore_plpython3u" to load this file. \quit --- make sure the prerequisite libraries are loaded -LOAD 'plpython3'; -SELECT NULL::hstore; - - CREATE FUNCTION hstore_to_plpython3(val internal) RETURNS internal LANGUAGE C STRICT IMMUTABLE AS 'MODULE_PATHNAME', 'hstore_to_plpython'; diff --git a/contrib/hstore_plpython/hstore_plpythonu--1.0.sql b/contrib/hstore_plpython/hstore_plpythonu--1.0.sql index d79bdc96d9..52832912ab 100644 --- a/contrib/hstore_plpython/hstore_plpythonu--1.0.sql +++ b/contrib/hstore_plpython/hstore_plpythonu--1.0.sql @@ -3,11 +3,6 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION hstore_plpythonu" to load this file. \quit --- make sure the prerequisite libraries are loaded -LOAD 'plpython2'; -- change to plpython3 if that ever becomes the default -SELECT NULL::hstore; - - CREATE FUNCTION hstore_to_plpython(val internal) RETURNS internal LANGUAGE C STRICT IMMUTABLE AS 'MODULE_PATHNAME'; diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 5add0b8a91..d2ab9e466e 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -475,10 +475,11 @@ sub mkvcbuild $plpython->AddReference($postgres); # Add transform modules dependent on plpython - AddTransformModule( + my $hstore_plpython = AddTransformModule( 'hstore_plpython' . $pymajorver, 'contrib/hstore_plpython', 'plpython' . $pymajorver, 'src/pl/plpython', 'hstore', 'contrib/hstore'); + $hstore_plpython->AddDefine('PLPYTHON_LIBNAME="plpython' . $pymajorver . '"'); AddTransformModule( 'ltree_plpython' . $pymajorver, 'contrib/ltree_plpython', 'plpython' . $pymajorver, 'src/pl/plpython', From 61633f79048d040aefeed68dcac0e0b996a06189 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Mon, 3 Oct 2016 22:11:36 -0700 Subject: [PATCH 267/871] Correct logical decoding restore behaviour for subtransactions. Before initializing iteration over a subtransaction's changes, the last few changes were not spilled to disk. That's correct if the transaction didn't spill to disk, but otherwise... This bug can lead to missed or misorderd subtransaction contents when they were spilled to disk. Move spilling of the remaining in-memory changes to ReorderBufferIterTXNInit(), where it can easily be applied to the top transaction and, if present, subtransactions. Since this code had too many bugs already, noticeably increase test coverage. Fixes: #14319 Reported-By: Huan Ruan Discussion: <20160909012610.20024.58169@wrigleys.postgresql.org> Backport: 9,4-, where logical decoding was added --- contrib/test_decoding/Makefile | 3 +- contrib/test_decoding/expected/spill.out | 256 ++++++++++++++++++ contrib/test_decoding/sql/spill.sql | 179 ++++++++++++ .../replication/logical/reorderbuffer.c | 13 +- 4 files changed, 445 insertions(+), 6 deletions(-) create mode 100644 contrib/test_decoding/expected/spill.out create mode 100644 contrib/test_decoding/sql/spill.sql diff --git a/contrib/test_decoding/Makefile b/contrib/test_decoding/Makefile index 309cb0b39a..a6641f5040 100644 --- a/contrib/test_decoding/Makefile +++ b/contrib/test_decoding/Makefile @@ -38,7 +38,8 @@ submake-test_decoding: $(MAKE) -C $(top_builddir)/contrib/test_decoding REGRESSCHECKS=ddl xact rewrite toast permissions decoding_in_xact \ - decoding_into_rel binary prepared replorigin time messages + decoding_into_rel binary prepared replorigin time messages \ + spill regresscheck: | submake-regress submake-test_decoding temp-install $(MKDIR_P) regression_output diff --git a/contrib/test_decoding/expected/spill.out b/contrib/test_decoding/expected/spill.out new file mode 100644 index 0000000000..363e9a3d17 --- /dev/null +++ b/contrib/test_decoding/expected/spill.out @@ -0,0 +1,256 @@ +-- predictability +SET synchronous_commit = on; +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +CREATE TABLE spill_test(data text); +-- consume DDL +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + data +------ +(0 rows) + +-- spilling main xact +BEGIN; +INSERT INTO spill_test SELECT 'serialize-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-----------------------+-------+---------------------------------------------------------------------+------------------------------------------------------------------------ + 'serialize-topbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-topbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-topbig--1:5000' +(1 row) + +-- spilling subxact, nothing in main +BEGIN; +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-----------------------+-------+---------------------------------------------------------------------+------------------------------------------------------------------------ + 'serialize-subbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig--1:5000' +(1 row) + +-- spilling subxact, spilling main xact +BEGIN; +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-----------------------------+-------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------- + 'serialize-subbig-topbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--1:5000' + 'serialize-subbig-topbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--2:10000' +(2 rows) + +-- spilling subxact, non-spilling main xact +BEGIN; +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topsmall--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topsmall--2:'||g.i FROM generate_series(5001, 5001) g(i); +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-------------------------------+-------+---------------------------------------------------------------------------------+--------------------------------------------------------------------------------- + 'serialize-subbig-topsmall--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topsmall--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topsmall--1:5000' + 'serialize-subbig-topsmall--2 | 1 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topsmall--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topsmall--2:5001' +(2 rows) + +-- not-spilling subxact, spilling main xact +BEGIN; +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-----------------------------+-------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------- + 'serialize-subbig-topbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--1:5000' + 'serialize-subbig-topbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--2:10000' +(2 rows) + +-- spilling main xact, spilling subxact +BEGIN; +INSERT INTO spill_test SELECT 'serialize-topbig-subbig--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-topbig-subbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +RELEASE SAVEPOINT s; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-----------------------------+-------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------- + 'serialize-topbig-subbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-topbig-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-topbig-subbig--1:5000' + 'serialize-topbig-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-topbig-subbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-topbig-subbig--2:10000' +(2 rows) + +-- spilling main xact, not spilling subxact +BEGIN; +INSERT INTO spill_test SELECT 'serialize-topbig-subsmall--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-topbig-subsmall--2:'||g.i FROM generate_series(5001, 5001) g(i); +RELEASE SAVEPOINT s; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-------------------------------+-------+---------------------------------------------------------------------------------+--------------------------------------------------------------------------------- + 'serialize-topbig-subsmall--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-topbig-subsmall--1:1' | table public.spill_test: INSERT: data[text]:'serialize-topbig-subsmall--1:5000' + 'serialize-topbig-subsmall--2 | 1 | table public.spill_test: INSERT: data[text]:'serialize-topbig-subsmall--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-topbig-subsmall--2:5001' +(2 rows) + +-- spilling subxact, followed by another spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-subbig-subbig--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-subbig-subbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +RELEASE SAVEPOINT s2; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-----------------------------+-------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------- + 'serialize-subbig-subbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-subbig--1:5000' + 'serialize-subbig-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-subbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-subbig--2:10000' +(2 rows) + +-- spilling subxact, followed by not spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-subbig-subsmall--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-subbig-subsmall--2:'||g.i FROM generate_series(5001, 5001) g(i); +RELEASE SAVEPOINT s2; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-------------------------------+-------+---------------------------------------------------------------------------------+--------------------------------------------------------------------------------- + 'serialize-subbig-subsmall--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-subsmall--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-subsmall--1:5000' + 'serialize-subbig-subsmall--2 | 1 | table public.spill_test: INSERT: data[text]:'serialize-subbig-subsmall--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-subsmall--2:5001' +(2 rows) + +-- not spilling subxact, followed by spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-subsmall-subbig--1:'||g.i FROM generate_series(1, 1) g(i); +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-subsmall-subbig--2:'||g.i FROM generate_series(2, 5001) g(i); +RELEASE SAVEPOINT s2; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-------------------------------+-------+------------------------------------------------------------------------------+--------------------------------------------------------------------------------- + 'serialize-subsmall-subbig--1 | 1 | table public.spill_test: INSERT: data[text]:'serialize-subsmall-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subsmall-subbig--1:1' + 'serialize-subsmall-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subsmall-subbig--2:2' | table public.spill_test: INSERT: data[text]:'serialize-subsmall-subbig--2:5001' +(2 rows) + +-- spilling subxact, containing another spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbig--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +RELEASE SAVEPOINT s2; +RELEASE SAVEPOINT s1; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +------------------------------------+-------+--------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------- + 'serialize-nested-subbig-subbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbig--1:5000' + 'serialize-nested-subbig-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbig--2:10000' +(2 rows) + +-- spilling subxact, containing a not spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subsmall--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subsmall--2:'||g.i FROM generate_series(5001, 5001) g(i); +RELEASE SAVEPOINT s2; +RELEASE SAVEPOINT s1; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +--------------------------------------+-------+----------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------- + 'serialize-nested-subbig-subsmall--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subsmall--1:1' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subsmall--1:5000' + 'serialize-nested-subbig-subsmall--2 | 1 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subsmall--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subsmall--2:5001' +(2 rows) + +-- not spilling subxact, containing a spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-nested-subsmall-subbig--1:'||g.i FROM generate_series(1, 1) g(i); +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-nested-subsmall-subbig--2:'||g.i FROM generate_series(2, 5001) g(i); +RELEASE SAVEPOINT s2; +RELEASE SAVEPOINT s1; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +--------------------------------------+-------+-------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------- + 'serialize-nested-subsmall-subbig--1 | 1 | table public.spill_test: INSERT: data[text]:'serialize-nested-subsmall-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-nested-subsmall-subbig--1:1' + 'serialize-nested-subsmall-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subsmall-subbig--2:2' | table public.spill_test: INSERT: data[text]:'serialize-nested-subsmall-subbig--2:5001' +(2 rows) + +-- not spilling subxact, containing a spilling subxact that aborts and one that commits +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbigabort--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbigabort--2:'||g.i FROM generate_series(5001, 10000) g(i); +ROLLBACK TO SAVEPOINT s2; +SAVEPOINT s3; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbigabort-subbig-3:'||g.i FROM generate_series(5001, 10000) g(i); +RELEASE SAVEPOINT s1; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + regexp_split_to_array | count | array_agg | array_agg +-----------------------------------------------+-------+-------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------- + 'serialize-nested-subbig-subbigabort--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbigabort--1:1' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbigabort--1:5000' + 'serialize-nested-subbig-subbigabort-subbig-3 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbigabort-subbig-3:5001' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbigabort-subbig-3:10000' +(2 rows) + +DROP TABLE spill_test; +SELECT pg_drop_replication_slot('regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + diff --git a/contrib/test_decoding/sql/spill.sql b/contrib/test_decoding/sql/spill.sql new file mode 100644 index 0000000000..358af0bfcd --- /dev/null +++ b/contrib/test_decoding/sql/spill.sql @@ -0,0 +1,179 @@ +-- predictability +SET synchronous_commit = on; + +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); + +CREATE TABLE spill_test(data text); + +-- consume DDL +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + +-- spilling main xact +BEGIN; +INSERT INTO spill_test SELECT 'serialize-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- spilling subxact, nothing in main +BEGIN; +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- spilling subxact, spilling main xact +BEGIN; +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- spilling subxact, non-spilling main xact +BEGIN; +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topsmall--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topsmall--2:'||g.i FROM generate_series(5001, 5001) g(i); +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- not-spilling subxact, spilling main xact +BEGIN; +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-subbig-topbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- spilling main xact, spilling subxact +BEGIN; +INSERT INTO spill_test SELECT 'serialize-topbig-subbig--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-topbig-subbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +RELEASE SAVEPOINT s; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- spilling main xact, not spilling subxact +BEGIN; +INSERT INTO spill_test SELECT 'serialize-topbig-subsmall--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s; +INSERT INTO spill_test SELECT 'serialize-topbig-subsmall--2:'||g.i FROM generate_series(5001, 5001) g(i); +RELEASE SAVEPOINT s; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- spilling subxact, followed by another spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-subbig-subbig--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-subbig-subbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +RELEASE SAVEPOINT s2; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- spilling subxact, followed by not spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-subbig-subsmall--1:'||g.i FROM generate_series(1, 5000) g(i); +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-subbig-subsmall--2:'||g.i FROM generate_series(5001, 5001) g(i); +RELEASE SAVEPOINT s2; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- not spilling subxact, followed by spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-subsmall-subbig--1:'||g.i FROM generate_series(1, 1) g(i); +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-subsmall-subbig--2:'||g.i FROM generate_series(2, 5001) g(i); +RELEASE SAVEPOINT s2; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- spilling subxact, containing another spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbig--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbig--2:'||g.i FROM generate_series(5001, 10000) g(i); +RELEASE SAVEPOINT s2; +RELEASE SAVEPOINT s1; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- spilling subxact, containing a not spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subsmall--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subsmall--2:'||g.i FROM generate_series(5001, 5001) g(i); +RELEASE SAVEPOINT s2; +RELEASE SAVEPOINT s1; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- not spilling subxact, containing a spilling subxact +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-nested-subsmall-subbig--1:'||g.i FROM generate_series(1, 1) g(i); +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-nested-subsmall-subbig--2:'||g.i FROM generate_series(2, 5001) g(i); +RELEASE SAVEPOINT s2; +RELEASE SAVEPOINT s1; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +-- not spilling subxact, containing a spilling subxact that aborts and one that commits +BEGIN; +SAVEPOINT s1; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbigabort--1:'||g.i FROM generate_series(1, 5000) g(i); +SAVEPOINT s2; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbigabort--2:'||g.i FROM generate_series(5001, 10000) g(i); +ROLLBACK TO SAVEPOINT s2; +SAVEPOINT s3; +INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbigabort-subbig-3:'||g.i FROM generate_series(5001, 10000) g(i); +RELEASE SAVEPOINT s1; +COMMIT; +SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)] +FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT' +GROUP BY 1 ORDER BY 1; + +DROP TABLE spill_test; + +SELECT pg_drop_replication_slot('regression_slot'); diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index e2a502c443..6ad7e7de76 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -935,8 +935,12 @@ ReorderBufferIterTXNInit(ReorderBuffer *rb, ReorderBufferTXN *txn) ReorderBufferChange *cur_change; if (txn->nentries != txn->nentries_mem) + { + /* serialize remaining changes */ + ReorderBufferSerializeTXN(rb, txn); ReorderBufferRestoreChanges(rb, txn, &state->entries[off].fd, &state->entries[off].segno); + } cur_change = dlist_head_element(ReorderBufferChange, node, &txn->changes); @@ -960,10 +964,13 @@ ReorderBufferIterTXNInit(ReorderBuffer *rb, ReorderBufferTXN *txn) ReorderBufferChange *cur_change; if (cur_txn->nentries != cur_txn->nentries_mem) + { + /* serialize remaining changes */ + ReorderBufferSerializeTXN(rb, cur_txn); ReorderBufferRestoreChanges(rb, cur_txn, &state->entries[off].fd, &state->entries[off].segno); - + } cur_change = dlist_head_element(ReorderBufferChange, node, &cur_txn->changes); @@ -1367,10 +1374,6 @@ ReorderBufferCommit(ReorderBuffer *rb, TransactionId xid, txn->origin_id = origin_id; txn->origin_lsn = origin_lsn; - /* serialize the last bunch of changes if we need start earlier anyway */ - if (txn->nentries_mem != txn->nentries) - ReorderBufferSerializeTXN(rb, txn); - /* * If this transaction didn't have any real changes in our database, it's * OK not to have a snapshot. Note that ReorderBufferCommitChild will have From c86c2d9d57c1e6c5fec860873734cccaf31df5b0 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 4 Oct 2016 09:46:43 +0300 Subject: [PATCH 268/871] Update comment. mergepreread()/mergeprereadone() don't exist anymore, the function that does roughly the same is now called mergereadnext(). --- src/backend/utils/sort/tuplesort.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index 20cfb0b139..5ff81ed95a 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -3083,14 +3083,14 @@ dumpbatch(Tuplesortstate *state, bool alltuples) * a call with no subsequent run actually written to destTape), we prefer * to write out a 0 tuple run. * - * mergepreread()/mergeprereadone() are prepared for 0 tuple runs, and - * will reliably mark the tape inactive for the merge when called from - * beginmerge(). This case is therefore similar to the case where - * mergeonerun() finds a dummy run for the tape, and so doesn't need to - * merge a run from the tape (or conceptually "merges" the dummy run, if - * you prefer). According to Knuth, Algorithm D "isn't strictly optimal" - * in its method of distribution and dummy run assignment; this edge case - * seems very unlikely to make that appreciably worse. + * mergereadnext() is prepared for 0 tuple runs, and will reliably mark + * the tape inactive for the merge when called from beginmerge(). This + * case is therefore similar to the case where mergeonerun() finds a dummy + * run for the tape, and so doesn't need to merge a run from the tape (or + * conceptually "merges" the dummy run, if you prefer). According to + * Knuth, Algorithm D "isn't strictly optimal" in its method of + * distribution and dummy run assignment; this edge case seems very + * unlikely to make that appreciably worse. */ Assert(state->status == TSS_BUILDRUNS); From 490ed1ebb9b26fb342a1e3240107092e9c5aec02 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 4 Oct 2016 09:38:43 -0400 Subject: [PATCH 269/871] Fix hstore_plpython for Python 3. In commit d51924be8, I overlooked the need to provide linkage for PLyUnicode_FromStringAndSize, because that's only used (and indeed only exists) in Python 3 builds. In light of the need to #if this item, rearrange the ordering of the code related to each function pointer, so as not to need more #if's than absolutely necessary. Per buildfarm. --- contrib/hstore_plpython/hstore_plpython.c | 34 ++++++++++++++--------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/contrib/hstore_plpython/hstore_plpython.c b/contrib/hstore_plpython/hstore_plpython.c index ce296ca486..b184324ebf 100644 --- a/contrib/hstore_plpython/hstore_plpython.c +++ b/contrib/hstore_plpython/hstore_plpython.c @@ -11,20 +11,22 @@ extern void _PG_init(void); /* Linkage to functions in plpython module */ typedef char *(*PLyObject_AsString_t) (PyObject *plrv); - static PLyObject_AsString_t PLyObject_AsString_p; +#if PY_MAJOR_VERSION >= 3 +typedef PyObject *(*PLyUnicode_FromStringAndSize_t) (const char *s, Py_ssize_t size); +static PLyUnicode_FromStringAndSize_t PLyUnicode_FromStringAndSize_p; +#endif /* Linkage to functions in hstore module */ typedef HStore *(*hstoreUpgrade_t) (Datum orig); -typedef int (*hstoreUniquePairs_t) (Pairs *a, int32 l, int32 *buflen); -typedef HStore *(*hstorePairs_t) (Pairs *pairs, int32 pcount, int32 buflen); -typedef size_t (*hstoreCheckKeyLen_t) (size_t len); -typedef size_t (*hstoreCheckValLen_t) (size_t len); - static hstoreUpgrade_t hstoreUpgrade_p; +typedef int (*hstoreUniquePairs_t) (Pairs *a, int32 l, int32 *buflen); static hstoreUniquePairs_t hstoreUniquePairs_p; +typedef HStore *(*hstorePairs_t) (Pairs *pairs, int32 pcount, int32 buflen); static hstorePairs_t hstorePairs_p; +typedef size_t (*hstoreCheckKeyLen_t) (size_t len); static hstoreCheckKeyLen_t hstoreCheckKeyLen_p; +typedef size_t (*hstoreCheckValLen_t) (size_t len); static hstoreCheckValLen_t hstoreCheckValLen_p; @@ -34,29 +36,34 @@ static hstoreCheckValLen_t hstoreCheckValLen_p; void _PG_init(void) { - /* These asserts verify that typedefs above match original declarations */ + /* Asserts verify that typedefs above match original declarations */ AssertVariableIsOfType(&PLyObject_AsString, PLyObject_AsString_t); - AssertVariableIsOfType(&hstoreUpgrade, hstoreUpgrade_t); - AssertVariableIsOfType(&hstoreUniquePairs, hstoreUniquePairs_t); - AssertVariableIsOfType(&hstorePairs, hstorePairs_t); - AssertVariableIsOfType(&hstoreCheckKeyLen, hstoreCheckKeyLen_t); - AssertVariableIsOfType(&hstoreCheckValLen, hstoreCheckValLen_t); - PLyObject_AsString_p = (PLyObject_AsString_t) load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLyObject_AsString", true, NULL); +#if PY_MAJOR_VERSION >= 3 + AssertVariableIsOfType(&PLyUnicode_FromStringAndSize, PLyUnicode_FromStringAndSize_t); + PLyUnicode_FromStringAndSize_p = (PLyUnicode_FromStringAndSize_t) + load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLyUnicode_FromStringAndSize", + true, NULL); +#endif + AssertVariableIsOfType(&hstoreUpgrade, hstoreUpgrade_t); hstoreUpgrade_p = (hstoreUpgrade_t) load_external_function("$libdir/hstore", "hstoreUpgrade", true, NULL); + AssertVariableIsOfType(&hstoreUniquePairs, hstoreUniquePairs_t); hstoreUniquePairs_p = (hstoreUniquePairs_t) load_external_function("$libdir/hstore", "hstoreUniquePairs", true, NULL); + AssertVariableIsOfType(&hstorePairs, hstorePairs_t); hstorePairs_p = (hstorePairs_t) load_external_function("$libdir/hstore", "hstorePairs", true, NULL); + AssertVariableIsOfType(&hstoreCheckKeyLen, hstoreCheckKeyLen_t); hstoreCheckKeyLen_p = (hstoreCheckKeyLen_t) load_external_function("$libdir/hstore", "hstoreCheckKeyLen", true, NULL); + AssertVariableIsOfType(&hstoreCheckValLen, hstoreCheckValLen_t); hstoreCheckValLen_p = (hstoreCheckValLen_t) load_external_function("$libdir/hstore", "hstoreCheckValLen", true, NULL); @@ -65,6 +72,7 @@ _PG_init(void) /* These defines must be after the module init function */ #define PLyObject_AsString PLyObject_AsString_p +#define PLyUnicode_FromStringAndSize PLyUnicode_FromStringAndSize_p #define hstoreUpgrade hstoreUpgrade_p #define hstoreUniquePairs hstoreUniquePairs_p #define hstorePairs hstorePairs_p From 6f3bd98ebfc008cbd676da777bb0b2376c4c4bfa Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 4 Oct 2016 10:50:13 -0400 Subject: [PATCH 270/871] Extend framework from commit 53be0b1ad to report latch waits. WaitLatch, WaitLatchOrSocket, and WaitEventSetWait now taken an additional wait_event_info parameter; legal values are defined in pgstat.h. This makes it possible to uniquely identify every point in the core code where we are waiting for a latch; extensions can pass WAIT_EXTENSION. Because latches were the major wait primitive not previously covered by this patch, it is now possible to see information in pg_stat_activity on a large number of important wait events not previously addressed, such as ClientRead, ClientWrite, and SyncRep. Unfortunately, many of the wait events added by this patch will fail to appear in pg_stat_activity because they're only used in background processes which don't currently appear in pg_stat_activity. We should fix this either by creating a separate view for such information, or else by deciding to include them in pg_stat_activity after all. Michael Paquier and Robert Haas, reviewed by Alexander Korotkov and Thomas Munro. --- contrib/postgres_fdw/connection.c | 3 +- doc/src/sgml/monitoring.sgml | 169 ++++++++++++++++++ src/backend/access/transam/parallel.c | 4 +- src/backend/access/transam/xlog.c | 7 +- src/backend/executor/nodeGather.c | 3 +- src/backend/libpq/be-secure-openssl.c | 4 +- src/backend/libpq/be-secure.c | 7 +- src/backend/libpq/pqmq.c | 4 +- src/backend/postmaster/autovacuum.c | 3 +- src/backend/postmaster/bgworker.c | 7 +- src/backend/postmaster/bgwriter.c | 5 +- src/backend/postmaster/checkpointer.c | 3 +- src/backend/postmaster/pgarch.c | 3 +- src/backend/postmaster/pgstat.c | 236 +++++++++++++++++++++++++- src/backend/postmaster/syslogger.c | 4 +- src/backend/postmaster/walwriter.c | 3 +- src/backend/replication/basebackup.c | 3 +- src/backend/replication/syncrep.c | 4 +- src/backend/replication/walreceiver.c | 7 +- src/backend/replication/walsender.c | 9 +- src/backend/storage/buffer/bufmgr.c | 7 +- src/backend/storage/ipc/latch.c | 18 +- src/backend/storage/ipc/shm_mq.c | 7 +- src/backend/storage/ipc/standby.c | 5 +- src/backend/storage/lmgr/lock.c | 3 - src/backend/storage/lmgr/lwlock.c | 6 +- src/backend/storage/lmgr/predicate.c | 3 +- src/backend/storage/lmgr/proc.c | 8 +- src/backend/utils/adt/misc.c | 4 +- src/include/pgstat.h | 99 +++++++++-- src/include/storage/latch.h | 9 +- src/include/storage/lwlock.h | 2 +- src/include/storage/proc.h | 2 +- src/test/modules/test_shm_mq/setup.c | 3 +- src/test/modules/test_shm_mq/test.c | 3 +- 35 files changed, 584 insertions(+), 83 deletions(-) diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c index 8ca1c1c898..9badfe6a7d 100644 --- a/contrib/postgres_fdw/connection.c +++ b/contrib/postgres_fdw/connection.c @@ -17,6 +17,7 @@ #include "access/xact.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/latch.h" #include "utils/hsearch.h" #include "utils/memutils.h" @@ -496,7 +497,7 @@ pgfdw_get_result(PGconn *conn, const char *query) wc = WaitLatchOrSocket(MyLatch, WL_LATCH_SET | WL_SOCKET_READABLE, PQsocket(conn), - -1L); + -1L, WAIT_EXTENSION); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f400785721..3de489e2f0 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -679,6 +679,42 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser buffer in question. + + + Activity: The server process is idle. This is used by + system processes waiting for activity in their main processing loop. + wait_event will identify the specific wait point. + + + + + Extension: The server process is waiting for activity + in an extension module. This category is useful for modules to + track custom waiting points. + + + + + Client: The server process is waiting for some activity + on a socket from user applications, and that the server expects + something to happen that is independent from its internal processes. + wait_event will identify the specific wait point. + + + + + IPC: The server process is waiting for some activity + from another process in the server. wait_event will + identify the specific wait point. + + + + + Timeout: The server process is waiting for a timeout + to expire. wait_event will identify the specific wait + point. + + @@ -1085,6 +1121,139 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser BufferPinWaiting to acquire a pin on a buffer. + + Activity + ArchiverMain + Waiting in main loop of the archiver process. + + + AutoVacuumMain + Waiting in main loop of autovacuum launcher process. + + + BgWriterHibernate + Waiting in background writer process, hibernating. + + + BgWriterMain + Waiting in main loop of background writer process background worker. + + + CheckpointerMain + Waiting in main loop of checkpointer process. + + + PgStatMain + Waiting in main loop of the statistics collector process. + + + RecoveryWalAll + Waiting for WAL from any kind of source (local, archive or stream) at recovery. + + + RecoveryWalStream + Waiting for WAL from a stream at recovery. + + + SysLoggerMain + Waiting in main loop of syslogger process. + + + WalReceiverMain + Waiting in main loop of WAL receiver process. + + + WalSenderMain + Waiting in main loop of WAL sender process. + + + WalWriterMain + Waiting in main loop of WAL writer process. + + + Client + ClientRead + Waiting to read data from the client. + + + ClientWrite + Waiting to write data from the client. + + + SSLOpenServer + Waiting for SSL while attempting connection. + + + WalReceiverWaitStart + Waiting for startup process to send initial data for streaming replication. + + + WalSenderWaitForWAL + Waiting for WAL to be flushed in WAL sender process. + + + WalSenderWriteData + Waiting for any activity when processing replies from WAL receiver in WAL sender process. + + + Extension + Extension + Waiting in an extension. + + + IPC + BgWorkerShutdown + Waiting for background worker to shut down. + + + BgWorkerStartup + Waiting for background worker to start up. + + + ExecuteGather + Waiting for activity from child process when executing Gather node. + + + MessageQueueInternal + Waiting for other process to be attached in shared message queue. + + + MessageQueuePutMessage + Waiting to write a protoocol message to a shared message queue. + + + MessageQueueReceive + Waiting to receive bytes from a shared message queue. + + + MessageQueueSend + Waiting to send bytes to a shared message queue. + + + ParallelFinish + Waiting for parallel workers to finish computing. + + + SafeSnapshot + Waiting for a snapshot for a READ ONLY DEFERRABLE transaction. + + + SyncRep + Waiting for confirmation from remote server during synchronous replication. + + + Timeout + BaseBackupThrottle + Waiting during base backup when throttling activity. + + + PgSleep + Waiting in process that called pg_sleep. + + + RecoveryApplyDelay + Waiting to apply WAL at recovery because it is delayed. +
diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c index cde0ed300f..59dc3949d8 100644 --- a/src/backend/access/transam/parallel.c +++ b/src/backend/access/transam/parallel.c @@ -24,6 +24,7 @@ #include "libpq/pqmq.h" #include "miscadmin.h" #include "optimizer/planmain.h" +#include "pgstat.h" #include "storage/ipc.h" #include "storage/sinval.h" #include "storage/spin.h" @@ -540,7 +541,8 @@ WaitForParallelWorkersToFinish(ParallelContext *pcxt) if (!anyone_alive) break; - WaitLatch(&MyProc->procLatch, WL_LATCH_SET, -1); + WaitLatch(&MyProc->procLatch, WL_LATCH_SET, -1, + WAIT_EVENT_PARALLEL_FINISH); ResetLatch(&MyProc->procLatch); } diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c1b9a97147..08c87f91be 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5827,7 +5827,8 @@ recoveryApplyDelay(XLogReaderState *record) WaitLatch(&XLogCtl->recoveryWakeupLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - secs * 1000L + microsecs / 1000); + secs * 1000L + microsecs / 1000, + WAIT_EVENT_RECOVERY_APPLY_DELAY); } return true; } @@ -11387,7 +11388,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, WaitLatch(&XLogCtl->recoveryWakeupLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - wait_time); + wait_time, WAIT_EVENT_RECOVERY_WAL_STREAM); ResetLatch(&XLogCtl->recoveryWakeupLatch); now = GetCurrentTimestamp(); } @@ -11550,7 +11551,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, */ WaitLatch(&XLogCtl->recoveryWakeupLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - 5000L); + 5000L, WAIT_EVENT_RECOVERY_WAL_ALL); ResetLatch(&XLogCtl->recoveryWakeupLatch); break; } diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c index 438d1b24fc..880ca62397 100644 --- a/src/backend/executor/nodeGather.c +++ b/src/backend/executor/nodeGather.c @@ -38,6 +38,7 @@ #include "executor/nodeSubplan.h" #include "executor/tqueue.h" #include "miscadmin.h" +#include "pgstat.h" #include "utils/memutils.h" #include "utils/rel.h" @@ -387,7 +388,7 @@ gather_readnext(GatherState *gatherstate) return NULL; /* Nothing to do except wait for developments. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EVENT_EXECUTE_GATHER); ResetLatch(MyLatch); nvisited = 0; } diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index fedb02cd82..668f217bba 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -60,6 +60,7 @@ #include "libpq/libpq.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/latch.h" #include "tcop/tcopprot.h" #include "utils/memutils.h" @@ -419,7 +420,8 @@ be_tls_open_server(Port *port) else waitfor = WL_SOCKET_WRITEABLE; - WaitLatchOrSocket(MyLatch, waitfor, port->sock, 0); + WaitLatchOrSocket(MyLatch, waitfor, port->sock, 0, + WAIT_EVENT_SSL_OPEN_SERVER); goto aloop; case SSL_ERROR_SYSCALL: if (r < 0) diff --git a/src/backend/libpq/be-secure.c b/src/backend/libpq/be-secure.c index cdd07d577b..b267507de9 100644 --- a/src/backend/libpq/be-secure.c +++ b/src/backend/libpq/be-secure.c @@ -33,6 +33,7 @@ #include "libpq/libpq.h" #include "miscadmin.h" +#include "pgstat.h" #include "tcop/tcopprot.h" #include "utils/memutils.h" #include "storage/ipc.h" @@ -146,7 +147,8 @@ secure_read(Port *port, void *ptr, size_t len) ModifyWaitEvent(FeBeWaitSet, 0, waitfor, NULL); - WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1); + WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, + WAIT_EVENT_CLIENT_READ); /* * If the postmaster has died, it's not safe to continue running, @@ -247,7 +249,8 @@ secure_write(Port *port, void *ptr, size_t len) ModifyWaitEvent(FeBeWaitSet, 0, waitfor, NULL); - WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1); + WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, + WAIT_EVENT_CLIENT_WRITE); /* See comments in secure_read. */ if (event.events & WL_POSTMASTER_DEATH) diff --git a/src/backend/libpq/pqmq.c b/src/backend/libpq/pqmq.c index bfe66c6c44..f93ccae148 100644 --- a/src/backend/libpq/pqmq.c +++ b/src/backend/libpq/pqmq.c @@ -17,6 +17,7 @@ #include "libpq/pqformat.h" #include "libpq/pqmq.h" #include "miscadmin.h" +#include "pgstat.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" @@ -171,7 +172,8 @@ mq_putmessage(char msgtype, const char *s, size_t len) if (result != SHM_MQ_WOULD_BLOCK) break; - WaitLatch(&MyProc->procLatch, WL_LATCH_SET, 0); + WaitLatch(&MyProc->procLatch, WL_LATCH_SET, 0, + WAIT_EVENT_MQ_PUT_MESSAGE); ResetLatch(&MyProc->procLatch); CHECK_FOR_INTERRUPTS(); } diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 1a92ca1deb..e3a6911fba 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -598,7 +598,8 @@ AutoVacLauncherMain(int argc, char *argv[]) */ rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - (nap.tv_sec * 1000L) + (nap.tv_usec / 1000L)); + (nap.tv_sec * 1000L) + (nap.tv_usec / 1000L), + WAIT_EVENT_AUTOVACUUM_MAIN); ResetLatch(MyLatch); diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 699c934240..028a9eed2d 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -18,6 +18,7 @@ #include "libpq/pqsignal.h" #include "postmaster/bgworker_internals.h" #include "postmaster/postmaster.h" +#include "pgstat.h" #include "storage/barrier.h" #include "storage/dsm.h" #include "storage/ipc.h" @@ -969,7 +970,8 @@ WaitForBackgroundWorkerStartup(BackgroundWorkerHandle *handle, pid_t *pidp) break; rc = WaitLatch(MyLatch, - WL_LATCH_SET | WL_POSTMASTER_DEATH, 0); + WL_LATCH_SET | WL_POSTMASTER_DEATH, 0, + WAIT_EVENT_BGWORKER_STARTUP); if (rc & WL_POSTMASTER_DEATH) { @@ -1008,7 +1010,8 @@ WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *handle) break; rc = WaitLatch(&MyProc->procLatch, - WL_LATCH_SET | WL_POSTMASTER_DEATH, 0); + WL_LATCH_SET | WL_POSTMASTER_DEATH, 0, + WAIT_EVENT_BGWORKER_SHUTDOWN); if (rc & WL_POSTMASTER_DEATH) { diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c index 10020349a2..c3f33561da 100644 --- a/src/backend/postmaster/bgwriter.c +++ b/src/backend/postmaster/bgwriter.c @@ -345,7 +345,7 @@ BackgroundWriterMain(void) */ rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - BgWriterDelay /* ms */ ); + BgWriterDelay /* ms */, WAIT_EVENT_BGWRITER_MAIN); /* * If no latch event and BgBufferSync says nothing's happening, extend @@ -372,7 +372,8 @@ BackgroundWriterMain(void) /* Sleep ... */ rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - BgWriterDelay * HIBERNATE_FACTOR); + BgWriterDelay * HIBERNATE_FACTOR, + WAIT_EVENT_BGWRITER_HIBERNATE); /* Reset the notification request in case we timed out */ StrategyNotifyBgWriter(-1); } diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index d702a4864d..397267c6b7 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -556,7 +556,8 @@ CheckpointerMain(void) rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - cur_timeout * 1000L /* convert to ms */ ); + cur_timeout * 1000L /* convert to ms */, + WAIT_EVENT_CHECKPOINTER_MAIN); /* * Emergency bailout if postmaster has died. This is to avoid the diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 1aa6466d67..62783d9259 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -390,7 +390,8 @@ pgarch_MainLoop(void) rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - timeout * 1000L); + timeout * 1000L, + WAIT_EVENT_ARCHIVER_MAIN); if (rc & WL_TIMEOUT) wakened = true; } diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 96578dcedb..8c9d06fdaa 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -276,6 +276,11 @@ static PgStat_TableStatus *get_tabstat_entry(Oid rel_id, bool isshared); static void pgstat_setup_memcxt(void); +static const char *pgstat_get_wait_activity(WaitEventActivity w); +static const char *pgstat_get_wait_client(WaitEventClient w); +static const char *pgstat_get_wait_ipc(WaitEventIPC w); +static const char *pgstat_get_wait_timeout(WaitEventTimeout w); + static void pgstat_setheader(PgStat_MsgHdr *hdr, StatMsgType mtype); static void pgstat_send(void *msg, int len); @@ -3131,15 +3136,14 @@ pgstat_read_current_status(void) const char * pgstat_get_wait_event_type(uint32 wait_event_info) { - uint8 classId; + uint32 classId; const char *event_type; /* report process as not waiting. */ if (wait_event_info == 0) return NULL; - wait_event_info = wait_event_info >> 24; - classId = wait_event_info & 0XFF; + classId = wait_event_info & 0xFF000000; switch (classId) { @@ -3155,6 +3159,21 @@ pgstat_get_wait_event_type(uint32 wait_event_info) case WAIT_BUFFER_PIN: event_type = "BufferPin"; break; + case WAIT_ACTIVITY: + event_type = "Activity"; + break; + case WAIT_CLIENT: + event_type = "Client"; + break; + case WAIT_EXTENSION: + event_type = "Extension"; + break; + case WAIT_IPC: + event_type = "IPC"; + break; + case WAIT_TIMEOUT: + event_type = "Timeout"; + break; default: event_type = "???"; break; @@ -3172,7 +3191,7 @@ pgstat_get_wait_event_type(uint32 wait_event_info) const char * pgstat_get_wait_event(uint32 wait_event_info) { - uint8 classId; + uint32 classId; uint16 eventId; const char *event_name; @@ -3180,9 +3199,8 @@ pgstat_get_wait_event(uint32 wait_event_info) if (wait_event_info == 0) return NULL; - eventId = wait_event_info & ((1 << 24) - 1); - wait_event_info = wait_event_info >> 24; - classId = wait_event_info & 0XFF; + classId = wait_event_info & 0xFF000000; + eventId = wait_event_info & 0x0000FFFF; switch (classId) { @@ -3196,6 +3214,37 @@ pgstat_get_wait_event(uint32 wait_event_info) case WAIT_BUFFER_PIN: event_name = "BufferPin"; break; + case WAIT_ACTIVITY: + { + WaitEventActivity w = (WaitEventActivity) wait_event_info; + + event_name = pgstat_get_wait_activity(w); + break; + } + case WAIT_CLIENT: + { + WaitEventClient w = (WaitEventClient) wait_event_info; + + event_name = pgstat_get_wait_client(w); + break; + } + case WAIT_EXTENSION: + event_name = "Extension"; + break; + case WAIT_IPC: + { + WaitEventIPC w = (WaitEventIPC) wait_event_info; + + event_name = pgstat_get_wait_ipc(w); + break; + } + case WAIT_TIMEOUT: + { + WaitEventTimeout w = (WaitEventTimeout) wait_event_info; + + event_name = pgstat_get_wait_timeout(w); + break; + } default: event_name = "unknown wait event"; break; @@ -3204,6 +3253,175 @@ pgstat_get_wait_event(uint32 wait_event_info) return event_name; } +/* ---------- + * pgstat_get_wait_activity() - + * + * Convert WaitEventActivity to string. + * ---------- + */ +static const char * +pgstat_get_wait_activity(WaitEventActivity w) +{ + const char *event_name = "unknown wait event"; + + switch (w) + { + case WAIT_EVENT_ARCHIVER_MAIN: + event_name = "ArchiverMain"; + break; + case WAIT_EVENT_AUTOVACUUM_MAIN: + event_name = "AutoVacuumMain"; + break; + case WAIT_EVENT_BGWRITER_HIBERNATE: + event_name = "BgWriterHibernate"; + break; + case WAIT_EVENT_BGWRITER_MAIN: + event_name = "BgWriterMain"; + break; + case WAIT_EVENT_CHECKPOINTER_MAIN: + event_name = "CheckpointerMain"; + break; + case WAIT_EVENT_PGSTAT_MAIN: + event_name = "PgStatMain"; + break; + case WAIT_EVENT_RECOVERY_WAL_ALL: + event_name = "RecoveryWalAll"; + break; + case WAIT_EVENT_RECOVERY_WAL_STREAM: + event_name = "RecoveryWalStream"; + break; + case WAIT_EVENT_SYSLOGGER_MAIN: + event_name = "SysLoggerMain"; + break; + case WAIT_EVENT_WAL_RECEIVER_MAIN: + event_name = "WalReceiverMain"; + break; + case WAIT_EVENT_WAL_SENDER_MAIN: + event_name = "WalSenderMain"; + break; + case WAIT_EVENT_WAL_WRITER_MAIN: + event_name = "WalWriterMain"; + break; + /* no default case, so that compiler will warn */ + } + + return event_name; +} + +/* ---------- + * pgstat_get_wait_client() - + * + * Convert WaitEventClient to string. + * ---------- + */ +static const char * +pgstat_get_wait_client(WaitEventClient w) +{ + const char *event_name = "unknown wait event"; + + switch (w) + { + case WAIT_EVENT_CLIENT_READ: + event_name = "ClientRead"; + break; + case WAIT_EVENT_CLIENT_WRITE: + event_name = "ClientWrite"; + break; + case WAIT_EVENT_SSL_OPEN_SERVER: + event_name = "SSLOpenServer"; + break; + case WAIT_EVENT_WAL_RECEIVER_WAIT_START: + event_name = "WalReceiverWaitStart"; + break; + case WAIT_EVENT_WAL_SENDER_WAIT_WAL: + event_name = "WalSenderWaitForWAL"; + break; + case WAIT_EVENT_WAL_SENDER_WRITE_DATA: + event_name = "WalSenderWriteData"; + break; + /* no default case, so that compiler will warn */ + } + + return event_name; +} + +/* ---------- + * pgstat_get_wait_ipc() - + * + * Convert WaitEventIPC to string. + * ---------- + */ +static const char * +pgstat_get_wait_ipc(WaitEventIPC w) +{ + const char *event_name = "unknown wait event"; + + switch (w) + { + case WAIT_EVENT_BGWORKER_SHUTDOWN: + event_name = "BgWorkerShutdown"; + break; + case WAIT_EVENT_BGWORKER_STARTUP: + event_name = "BgWorkerStartup"; + break; + case WAIT_EVENT_EXECUTE_GATHER: + event_name = "ExecuteGather"; + break; + case WAIT_EVENT_MQ_INTERNAL: + event_name = "MessageQueueInternal"; + break; + case WAIT_EVENT_MQ_PUT_MESSAGE: + event_name = "MessageQueuePutMessage"; + break; + case WAIT_EVENT_MQ_RECEIVE: + event_name = "MessageQueueReceive"; + break; + case WAIT_EVENT_MQ_SEND: + event_name = "MessageQueueSend"; + break; + case WAIT_EVENT_PARALLEL_FINISH: + event_name = "ParallelFinish"; + break; + case WAIT_EVENT_SAFE_SNAPSHOT: + event_name = "SafeSnapshot"; + break; + case WAIT_EVENT_SYNC_REP: + event_name = "SyncRep"; + break; + /* no default case, so that compiler will warn */ + } + + return event_name; +} + +/* ---------- + * pgstat_get_wait_timeout() - + * + * Convert WaitEventTimeout to string. + * ---------- + */ +static const char * +pgstat_get_wait_timeout(WaitEventTimeout w) +{ + const char *event_name = "unknown wait event"; + + switch (w) + { + case WAIT_EVENT_BASE_BACKUP_THROTTLE: + event_name = "BaseBackupThrottle"; + break; + case WAIT_EVENT_PG_SLEEP: + event_name = "PgSleep"; + break; + case WAIT_EVENT_RECOVERY_APPLY_DELAY: + event_name = "RecoveryApplyDelay"; + break; + /* no default case, so that compiler will warn */ + } + + return event_name; +} + /* ---------- * pgstat_get_backend_current_activity() - * @@ -3684,8 +3902,8 @@ PgstatCollectorMain(int argc, char *argv[]) #ifndef WIN32 wr = WaitLatchOrSocket(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH | WL_SOCKET_READABLE, - pgStatSock, - -1L); + pgStatSock, -1L, + WAIT_EVENT_PGSTAT_MAIN); #else /* diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index e7e488a236..af7136760a 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -35,6 +35,7 @@ #include "libpq/pqsignal.h" #include "miscadmin.h" #include "nodes/pg_list.h" +#include "pgstat.h" #include "pgtime.h" #include "postmaster/fork_process.h" #include "postmaster/postmaster.h" @@ -424,7 +425,8 @@ SysLoggerMain(int argc, char *argv[]) rc = WaitLatchOrSocket(MyLatch, WL_LATCH_SET | WL_SOCKET_READABLE | cur_flags, syslogPipe[0], - cur_timeout); + cur_timeout, + WAIT_EVENT_SYSLOGGER_MAIN); if (rc & WL_SOCKET_READABLE) { diff --git a/src/backend/postmaster/walwriter.c b/src/backend/postmaster/walwriter.c index 11ec56aebb..67dcff63b1 100644 --- a/src/backend/postmaster/walwriter.c +++ b/src/backend/postmaster/walwriter.c @@ -290,7 +290,8 @@ WalWriterMain(void) rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - cur_timeout); + cur_timeout, + WAIT_EVENT_WAL_WRITER_MAIN); /* * Emergency bailout if postmaster has died. This is to avoid the diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 1eabaef492..fa75930c9f 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -1364,7 +1364,8 @@ throttle(size_t increment) */ wait_result = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - (long) (sleep / 1000)); + (long) (sleep / 1000), + WAIT_EVENT_BASE_BACKUP_THROTTLE); if (wait_result & WL_LATCH_SET) CHECK_FOR_INTERRUPTS(); diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index b442d061ec..ac29f567c3 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -61,6 +61,7 @@ #include "access/xact.h" #include "miscadmin.h" +#include "pgstat.h" #include "replication/syncrep.h" #include "replication/walsender.h" #include "replication/walsender_private.h" @@ -258,7 +259,8 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) * Wait on latch. Any condition that should wake us up will set the * latch, so no need for timeout. */ - WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1); + WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1, + WAIT_EVENT_SYNC_REP); } /* diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 413ee3a5c1..eed6effeeb 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -55,6 +55,7 @@ #include "libpq/pqformat.h" #include "libpq/pqsignal.h" #include "miscadmin.h" +#include "pgstat.h" #include "replication/walreceiver.h" #include "replication/walsender.h" #include "storage/ipc.h" @@ -486,7 +487,8 @@ WalReceiverMain(void) WL_POSTMASTER_DEATH | WL_SOCKET_READABLE | WL_TIMEOUT | WL_LATCH_SET, wait_fd, - NAPTIME_PER_CYCLE); + NAPTIME_PER_CYCLE, + WAIT_EVENT_WAL_RECEIVER_MAIN); if (rc & WL_LATCH_SET) { ResetLatch(&walrcv->latch); @@ -685,7 +687,8 @@ WalRcvWaitForStartPosition(XLogRecPtr *startpoint, TimeLineID *startpointTLI) } SpinLockRelease(&walrcv->mutex); - WaitLatch(&walrcv->latch, WL_LATCH_SET | WL_POSTMASTER_DEATH, 0); + WaitLatch(&walrcv->latch, WL_LATCH_SET | WL_POSTMASTER_DEATH, 0, + WAIT_EVENT_WAL_RECEIVER_WAIT_START); } if (update_process_title) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index c7743da034..0f3ced250c 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1146,7 +1146,8 @@ WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, /* Sleep until something happens or we time out */ WaitLatchOrSocket(MyLatch, wakeEvents, - MyProcPort->sock, sleeptime); + MyProcPort->sock, sleeptime, + WAIT_EVENT_WAL_SENDER_WRITE_DATA); } /* reactivate latch so WalSndLoop knows to continue */ @@ -1272,7 +1273,8 @@ WalSndWaitForWal(XLogRecPtr loc) /* Sleep until something happens or we time out */ WaitLatchOrSocket(MyLatch, wakeEvents, - MyProcPort->sock, sleeptime); + MyProcPort->sock, sleeptime, + WAIT_EVENT_WAL_SENDER_WAIT_WAL); } /* reactivate latch so WalSndLoop knows to continue */ @@ -1924,7 +1926,8 @@ WalSndLoop(WalSndSendDataCallback send_data) /* Sleep until something happens or we time out */ WaitLatchOrSocket(MyLatch, wakeEvents, - MyProcPort->sock, sleeptime); + MyProcPort->sock, sleeptime, + WAIT_EVENT_WAL_SENDER_MAIN); } } return; diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 90804a3c53..91dc24c301 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -3635,9 +3635,6 @@ LockBufferForCleanup(Buffer buffer) UnlockBufHdr(bufHdr, buf_state); LockBuffer(buffer, BUFFER_LOCK_UNLOCK); - /* Report the wait */ - pgstat_report_wait_start(WAIT_BUFFER_PIN, 0); - /* Wait to be signaled by UnpinBuffer() */ if (InHotStandby) { @@ -3649,9 +3646,7 @@ LockBufferForCleanup(Buffer buffer) SetStartupBufferPinWaitBufId(-1); } else - ProcWaitForSignal(); - - pgstat_report_wait_end(); + ProcWaitForSignal(WAIT_BUFFER_PIN); /* * Remove flag marking us as waiter. Normally this will not be set diff --git a/src/backend/storage/ipc/latch.c b/src/backend/storage/ipc/latch.c index 9def8a12d3..8488f944de 100644 --- a/src/backend/storage/ipc/latch.c +++ b/src/backend/storage/ipc/latch.c @@ -55,6 +55,7 @@ #endif #include "miscadmin.h" +#include "pgstat.h" #include "portability/instr_time.h" #include "postmaster/postmaster.h" #include "storage/barrier.h" @@ -297,9 +298,11 @@ DisownLatch(volatile Latch *latch) * we return all of them in one call, but we will return at least one. */ int -WaitLatch(volatile Latch *latch, int wakeEvents, long timeout) +WaitLatch(volatile Latch *latch, int wakeEvents, long timeout, + uint32 wait_event_info) { - return WaitLatchOrSocket(latch, wakeEvents, PGINVALID_SOCKET, timeout); + return WaitLatchOrSocket(latch, wakeEvents, PGINVALID_SOCKET, timeout, + wait_event_info); } /* @@ -316,7 +319,7 @@ WaitLatch(volatile Latch *latch, int wakeEvents, long timeout) */ int WaitLatchOrSocket(volatile Latch *latch, int wakeEvents, pgsocket sock, - long timeout) + long timeout, uint32 wait_event_info) { int ret = 0; int rc; @@ -344,7 +347,7 @@ WaitLatchOrSocket(volatile Latch *latch, int wakeEvents, pgsocket sock, AddWaitEventToSet(set, ev, sock, NULL, NULL); } - rc = WaitEventSetWait(set, timeout, &event, 1); + rc = WaitEventSetWait(set, timeout, &event, 1, wait_event_info); if (rc == 0) ret |= WL_TIMEOUT; @@ -863,7 +866,8 @@ WaitEventAdjustWin32(WaitEventSet *set, WaitEvent *event) */ int WaitEventSetWait(WaitEventSet *set, long timeout, - WaitEvent *occurred_events, int nevents) + WaitEvent *occurred_events, int nevents, + uint32 wait_event_info) { int returned_events = 0; instr_time start_time; @@ -883,6 +887,8 @@ WaitEventSetWait(WaitEventSet *set, long timeout, cur_timeout = timeout; } + pgstat_report_wait_start(wait_event_info); + #ifndef WIN32 waiting = true; #else @@ -960,6 +966,8 @@ WaitEventSetWait(WaitEventSet *set, long timeout, waiting = false; #endif + pgstat_report_wait_end(); + return returned_events; } diff --git a/src/backend/storage/ipc/shm_mq.c b/src/backend/storage/ipc/shm_mq.c index 5b32782022..bfb67038ad 100644 --- a/src/backend/storage/ipc/shm_mq.c +++ b/src/backend/storage/ipc/shm_mq.c @@ -19,6 +19,7 @@ #include "postgres.h" #include "miscadmin.h" +#include "pgstat.h" #include "postmaster/bgworker.h" #include "storage/procsignal.h" #include "storage/shm_mq.h" @@ -894,7 +895,7 @@ shm_mq_send_bytes(shm_mq_handle *mqh, Size nbytes, const void *data, * at top of loop, because setting an already-set latch is much * cheaper than setting one that has been reset. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EVENT_MQ_SEND); /* Reset the latch so we don't spin. */ ResetLatch(MyLatch); @@ -991,7 +992,7 @@ shm_mq_receive_bytes(shm_mq *mq, Size bytes_needed, bool nowait, * loop, because setting an already-set latch is much cheaper than * setting one that has been reset. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EVENT_MQ_RECEIVE); /* Reset the latch so we don't spin. */ ResetLatch(MyLatch); @@ -1090,7 +1091,7 @@ shm_mq_wait_internal(volatile shm_mq *mq, PGPROC *volatile * ptr, } /* Wait to be signalled. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EVENT_MQ_INTERNAL); /* Reset the latch so we don't spin. */ ResetLatch(MyLatch); diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c index 547f1a88fe..fb887b3230 100644 --- a/src/backend/storage/ipc/standby.c +++ b/src/backend/storage/ipc/standby.c @@ -22,6 +22,7 @@ #include "access/xlog.h" #include "access/xloginsert.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" #include "storage/proc.h" @@ -389,7 +390,7 @@ ResolveRecoveryConflictWithLock(LOCKTAG locktag) } /* Wait to be signaled by the release of the Relation Lock */ - ProcWaitForSignal(); + ProcWaitForSignal(WAIT_LOCK | locktag.locktag_type); /* * Clear any timeout requests established above. We assume here that the @@ -469,7 +470,7 @@ ResolveRecoveryConflictWithBufferPin(void) } /* Wait to be signaled by UnpinBuffer() */ - ProcWaitForSignal(); + ProcWaitForSignal(WAIT_BUFFER_PIN); /* * Clear any timeout requests established above. We assume here that the diff --git a/src/backend/storage/lmgr/lock.c b/src/backend/storage/lmgr/lock.c index dba3809e74..71a4dd4736 100644 --- a/src/backend/storage/lmgr/lock.c +++ b/src/backend/storage/lmgr/lock.c @@ -1676,7 +1676,6 @@ WaitOnLock(LOCALLOCK *locallock, ResourceOwner owner) set_ps_display(new_status, false); new_status[len] = '\0'; /* truncate off " waiting" */ } - pgstat_report_wait_start(WAIT_LOCK, locallock->tag.lock.locktag_type); awaitedLock = locallock; awaitedOwner = owner; @@ -1724,7 +1723,6 @@ WaitOnLock(LOCALLOCK *locallock, ResourceOwner owner) /* In this path, awaitedLock remains set until LockErrorCleanup */ /* Report change to non-waiting status */ - pgstat_report_wait_end(); if (update_process_title) { set_ps_display(new_status, false); @@ -1739,7 +1737,6 @@ WaitOnLock(LOCALLOCK *locallock, ResourceOwner owner) awaitedLock = NULL; /* Report change to non-waiting status */ - pgstat_report_wait_end(); if (update_process_title) { set_ps_display(new_status, false); diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 9d08de75ae..a90b54ac86 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -732,9 +732,9 @@ LWLockReportWaitStart(LWLock *lock) int lockId = T_ID(lock); if (lock->tranche == 0) - pgstat_report_wait_start(WAIT_LWLOCK_NAMED, (uint16) lockId); + pgstat_report_wait_start(WAIT_LWLOCK_NAMED | (uint16) lockId); else - pgstat_report_wait_start(WAIT_LWLOCK_TRANCHE, lock->tranche); + pgstat_report_wait_start(WAIT_LWLOCK_TRANCHE | lock->tranche); } /* @@ -750,7 +750,7 @@ LWLockReportWaitEnd(void) * Return an identifier for an LWLock based on the wait class and event. */ const char * -GetLWLockIdentifier(uint8 classId, uint16 eventId) +GetLWLockIdentifier(uint32 classId, uint16 eventId) { if (classId == WAIT_LWLOCK_NAMED) return MainLWLockNames[eventId]; diff --git a/src/backend/storage/lmgr/predicate.c b/src/backend/storage/lmgr/predicate.c index 4064b2033c..24ed21b487 100644 --- a/src/backend/storage/lmgr/predicate.c +++ b/src/backend/storage/lmgr/predicate.c @@ -192,6 +192,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/bufmgr.h" #include "storage/predicate.h" #include "storage/predicate_internals.h" @@ -1518,7 +1519,7 @@ GetSafeSnapshot(Snapshot origSnapshot) SxactIsROUnsafe(MySerializableXact))) { LWLockRelease(SerializableXactHashLock); - ProcWaitForSignal(); + ProcWaitForSignal(WAIT_EVENT_SAFE_SNAPSHOT); LWLockAcquire(SerializableXactHashLock, LW_EXCLUSIVE); } MySerializableXact->flags &= ~SXACT_FLAG_DEFERRABLE_WAITING; diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 33e7023656..dd76094bcd 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -39,6 +39,7 @@ #include "access/twophase.h" #include "access/xact.h" #include "miscadmin.h" +#include "pgstat.h" #include "postmaster/autovacuum.h" #include "replication/slot.h" #include "replication/syncrep.h" @@ -1212,7 +1213,8 @@ ProcSleep(LOCALLOCK *locallock, LockMethod lockMethodTable) } else { - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, + WAIT_LOCK | locallock->tag.lock.locktag_type); ResetLatch(MyLatch); /* check for deadlocks first, as that's probably log-worthy */ if (got_deadlock_timeout) @@ -1722,9 +1724,9 @@ CheckDeadLockAlert(void) * wait again if not. */ void -ProcWaitForSignal(void) +ProcWaitForSignal(uint32 wait_event_info) { - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, wait_event_info); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c index 5e705e9308..0da051a2f2 100644 --- a/src/backend/utils/adt/misc.c +++ b/src/backend/utils/adt/misc.c @@ -29,6 +29,7 @@ #include "common/keywords.h" #include "funcapi.h" #include "miscadmin.h" +#include "pgstat.h" #include "parser/scansup.h" #include "postmaster/syslogger.h" #include "rewrite/rewriteHandler.h" @@ -560,7 +561,8 @@ pg_sleep(PG_FUNCTION_ARGS) (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT, - delay_ms); + delay_ms, + WAIT_EVENT_PG_SLEEP); ResetLatch(MyLatch); } diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 0c98c59e72..b530c01984 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -715,15 +715,91 @@ typedef enum BackendState * Wait Classes * ---------- */ -typedef enum WaitClass +#define WAIT_LWLOCK_NAMED 0x01000000U +#define WAIT_LWLOCK_TRANCHE 0x02000000U +#define WAIT_LOCK 0x03000000U +#define WAIT_BUFFER_PIN 0x04000000U +#define WAIT_ACTIVITY 0x05000000U +#define WAIT_CLIENT 0x06000000U +#define WAIT_EXTENSION 0x07000000U +#define WAIT_IPC 0x08000000U +#define WAIT_TIMEOUT 0x09000000U + +/* ---------- + * Wait Events - Activity + * + * Use this category when a process is waiting because it has no work to do, + * unless the "Client" or "Timeout" category describes the situation better. + * Typically, this should only be used for background processes. + * ---------- + */ +typedef enum { - WAIT_UNDEFINED, - WAIT_LWLOCK_NAMED, - WAIT_LWLOCK_TRANCHE, - WAIT_LOCK, - WAIT_BUFFER_PIN -} WaitClass; + WAIT_EVENT_ARCHIVER_MAIN = WAIT_ACTIVITY, + WAIT_EVENT_AUTOVACUUM_MAIN, + WAIT_EVENT_BGWRITER_HIBERNATE, + WAIT_EVENT_BGWRITER_MAIN, + WAIT_EVENT_CHECKPOINTER_MAIN, + WAIT_EVENT_PGSTAT_MAIN, + WAIT_EVENT_RECOVERY_WAL_ALL, + WAIT_EVENT_RECOVERY_WAL_STREAM, + WAIT_EVENT_SYSLOGGER_MAIN, + WAIT_EVENT_WAL_RECEIVER_MAIN, + WAIT_EVENT_WAL_SENDER_MAIN, + WAIT_EVENT_WAL_WRITER_MAIN, +} WaitEventActivity; +/* ---------- + * Wait Events - Client + * + * Use this category when a process is waiting to send data to or receive data + * from the frontend process to which it is connected. This is never used for + * a background process, which has no client connection. + * ---------- + */ +typedef enum +{ + WAIT_EVENT_CLIENT_READ = WAIT_CLIENT, + WAIT_EVENT_CLIENT_WRITE, + WAIT_EVENT_SSL_OPEN_SERVER, + WAIT_EVENT_WAL_RECEIVER_WAIT_START, + WAIT_EVENT_WAL_SENDER_WAIT_WAL, + WAIT_EVENT_WAL_SENDER_WRITE_DATA, +} WaitEventClient; + +/* ---------- + * Wait Events - IPC + * + * Use this category when a process cannot complete the work it is doing because + * it is waiting for a notification from another process. + * ---------- + */ +typedef enum +{ + WAIT_EVENT_BGWORKER_SHUTDOWN = WAIT_IPC, + WAIT_EVENT_BGWORKER_STARTUP, + WAIT_EVENT_EXECUTE_GATHER, + WAIT_EVENT_MQ_INTERNAL, + WAIT_EVENT_MQ_PUT_MESSAGE, + WAIT_EVENT_MQ_RECEIVE, + WAIT_EVENT_MQ_SEND, + WAIT_EVENT_PARALLEL_FINISH, + WAIT_EVENT_SAFE_SNAPSHOT, + WAIT_EVENT_SYNC_REP +} WaitEventIPC; + +/* ---------- + * Wait Events - Timeout + * + * Use this category when a process is waiting for a timeout to expire. + * ---------- + */ +typedef enum +{ + WAIT_EVENT_BASE_BACKUP_THROTTLE = WAIT_TIMEOUT, + WAIT_EVENT_PG_SLEEP, + WAIT_EVENT_RECOVERY_APPLY_DELAY +} WaitEventTimeout; /* ---------- * Command type for progress reporting purposes @@ -1018,23 +1094,18 @@ extern void pgstat_initstats(Relation rel); * ---------- */ static inline void -pgstat_report_wait_start(uint8 classId, uint16 eventId) +pgstat_report_wait_start(uint32 wait_event_info) { volatile PGPROC *proc = MyProc; - uint32 wait_event_val; if (!pgstat_track_activities || !proc) return; - wait_event_val = classId; - wait_event_val <<= 24; - wait_event_val |= eventId; - /* * Since this is a four-byte field which is always read and written as * four-bytes, updates are atomic. */ - proc->wait_event_info = wait_event_val; + proc->wait_event_info = wait_event_info; } /* ---------- diff --git a/src/include/storage/latch.h b/src/include/storage/latch.h index 5179ecc0db..e96e88f2fa 100644 --- a/src/include/storage/latch.h +++ b/src/include/storage/latch.h @@ -155,10 +155,13 @@ extern int AddWaitEventToSet(WaitEventSet *set, uint32 events, pgsocket fd, Latch *latch, void *user_data); extern void ModifyWaitEvent(WaitEventSet *set, int pos, uint32 events, Latch *latch); -extern int WaitEventSetWait(WaitEventSet *set, long timeout, WaitEvent *occurred_events, int nevents); -extern int WaitLatch(volatile Latch *latch, int wakeEvents, long timeout); +extern int WaitEventSetWait(WaitEventSet *set, long timeout, + WaitEvent *occurred_events, int nevents, + uint32 wait_event_info); +extern int WaitLatch(volatile Latch *latch, int wakeEvents, long timeout, + uint32 wait_event_info); extern int WaitLatchOrSocket(volatile Latch *latch, int wakeEvents, - pgsocket sock, long timeout); + pgsocket sock, long timeout, uint32 wait_event_info); /* * Unix implementation uses SIGUSR1 for inter-process signaling. diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 18931eb046..9a2d86975c 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -184,7 +184,7 @@ extern Size LWLockShmemSize(void); extern void CreateLWLocks(void); extern void InitLWLockAccess(void); -extern const char *GetLWLockIdentifier(uint8 classId, uint16 eventId); +extern const char *GetLWLockIdentifier(uint32 classId, uint16 eventId); /* * Extensions (or core code) can obtain an LWLocks by calling diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index f576f052df..7dc8dac6d1 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -291,7 +291,7 @@ extern void CheckDeadLockAlert(void); extern bool IsWaitingForLock(void); extern void LockErrorCleanup(void); -extern void ProcWaitForSignal(void); +extern void ProcWaitForSignal(uint32 wait_event_info); extern void ProcSendSignal(int pid); extern void BecomeLockGroupLeader(void); diff --git a/src/test/modules/test_shm_mq/setup.c b/src/test/modules/test_shm_mq/setup.c index 143df4eb65..cb86f3c37d 100644 --- a/src/test/modules/test_shm_mq/setup.c +++ b/src/test/modules/test_shm_mq/setup.c @@ -16,6 +16,7 @@ #include "postgres.h" #include "miscadmin.h" +#include "pgstat.h" #include "postmaster/bgworker.h" #include "storage/procsignal.h" #include "storage/shm_toc.h" @@ -279,7 +280,7 @@ wait_for_workers_to_become_ready(worker_state *wstate, } /* Wait to be signalled. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EXTENSION); /* Reset the latch so we don't spin. */ ResetLatch(MyLatch); diff --git a/src/test/modules/test_shm_mq/test.c b/src/test/modules/test_shm_mq/test.c index dd34bc7e7f..bf11137a96 100644 --- a/src/test/modules/test_shm_mq/test.c +++ b/src/test/modules/test_shm_mq/test.c @@ -15,6 +15,7 @@ #include "fmgr.h" #include "miscadmin.h" +#include "pgstat.h" #include "test_shm_mq.h" @@ -230,7 +231,7 @@ test_shm_mq_pipelined(PG_FUNCTION_ARGS) * have read or written data and therefore there may now be work * for us to do. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EXTENSION); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } From 976a1ce91008944121d0b9d2b84c9644c36f7fbb Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 4 Oct 2016 11:18:43 -0400 Subject: [PATCH 271/871] Adjust worker_spi for 6f3bd98ebfc008cbd676da777bb0b2376c4c4bfa. --- src/test/modules/worker_spi/worker_spi.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/modules/worker_spi/worker_spi.c b/src/test/modules/worker_spi/worker_spi.c index 7c9a3eb67e..f8395bfb57 100644 --- a/src/test/modules/worker_spi/worker_spi.c +++ b/src/test/modules/worker_spi/worker_spi.c @@ -227,7 +227,8 @@ worker_spi_main(Datum main_arg) */ rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - worker_spi_naptime * 1000L); + worker_spi_naptime * 1000L, + WAIT_EXTENSION); ResetLatch(MyLatch); /* emergency bailout if postmaster has died */ From 23843dcb60f941786ab57fec804234bfadd5d17f Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 4 Oct 2016 11:49:09 -0400 Subject: [PATCH 272/871] Remove trailing commas from enums. Buildfarm member mylodon doesn't like them. Actually, I don't like them either, but I failed to notice these before pushing commit 6f3bd98ebfc008cbd676da777bb0b2376c4c4bfa. --- src/include/pgstat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/pgstat.h b/src/include/pgstat.h index b530c01984..27be5493fd 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -746,7 +746,7 @@ typedef enum WAIT_EVENT_SYSLOGGER_MAIN, WAIT_EVENT_WAL_RECEIVER_MAIN, WAIT_EVENT_WAL_SENDER_MAIN, - WAIT_EVENT_WAL_WRITER_MAIN, + WAIT_EVENT_WAL_WRITER_MAIN } WaitEventActivity; /* ---------- @@ -764,7 +764,7 @@ typedef enum WAIT_EVENT_SSL_OPEN_SERVER, WAIT_EVENT_WAL_RECEIVER_WAIT_START, WAIT_EVENT_WAL_SENDER_WAIT_WAL, - WAIT_EVENT_WAL_SENDER_WRITE_DATA, + WAIT_EVENT_WAL_SENDER_WRITE_DATA } WaitEventClient; /* ---------- From d4fca5e6c7363ba6ee4de7b8d72d68064fa864ca Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 4 Oct 2016 19:16:00 +0300 Subject: [PATCH 273/871] Fix another outdated comment. Preloading is done by logtape.c now. --- src/backend/utils/sort/tuplesort.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index 5ff81ed95a..dd83e7a8f2 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -2882,9 +2882,8 @@ mergeonerun(Tuplesortstate *state) * beginmerge - initialize for a merge pass * * We decrease the counts of real and dummy runs for each tape, and mark - * which tapes contain active input runs in mergeactive[]. Then, load - * as many tuples as we can from each active input tape, and finally - * fill the merge heap with the first tuple from each active tape. + * which tapes contain active input runs in mergeactive[]. Then, fill the + * merge heap with the first tuple from each active tape. */ static void beginmerge(Tuplesortstate *state) From 9445d1121d28d0a8ae32c907320b5d6ff985077b Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 4 Oct 2016 12:17:14 -0400 Subject: [PATCH 274/871] Fix Windows compile break in 6f3bd98ebfc008cbd676da777bb0b2376c4c4bfa. --- src/backend/postmaster/pgstat.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 8c9d06fdaa..5c6cb6b266 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -3919,7 +3919,8 @@ PgstatCollectorMain(int argc, char *argv[]) wr = WaitLatchOrSocket(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH | WL_SOCKET_READABLE | WL_TIMEOUT, pgStatSock, - 2 * 1000L /* msec */ ); + 2 * 1000L /* msec */, + WAIT_EVENT_PGSTAT_MAIN); #endif /* From 6c9c95ed1b89add3a78d115a90e92d765b4c859a Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 4 Oct 2016 13:14:19 -0400 Subject: [PATCH 275/871] Fix another Windows compile break. Commit 6f3bd98ebfc008cbd676da777bb0b2376c4c4bfa is still making the buildfarm unhappy. This time it's mastodon that is complaining. --- src/backend/postmaster/syslogger.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index af7136760a..fd62d660c4 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -477,7 +477,8 @@ SysLoggerMain(int argc, char *argv[]) (void) WaitLatch(MyLatch, WL_LATCH_SET | cur_flags, - cur_timeout); + cur_timeout, + WAIT_EVENT_SYSLOGGER_MAIN); EnterCriticalSection(&sysloggerSection); #endif /* WIN32 */ From 46ddbbb1177a7e6ce5a4fe0ef8fa8ac49f36d0bb Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 4 Oct 2016 15:23:02 -0400 Subject: [PATCH 276/871] Improve (I hope) our autolocation of the Python shared library. Older versions of Python produce garbage (or at least useless) values of get_config_vars('LDLIBRARY'). Newer versions produce garbage (or at least useless) values of get_config_vars('SO'), which was defeating our configure logic that attempted to identify where the Python shlib really is. The net result, at least with a stock Python 3.5 installation on macOS, was that we were linking against a static library in the mistaken belief that it was a shared library. This managed to work, if you count statically absorbing libpython into plpython.so as working. But it no longer works as of commit d51924be8, because now we get separate static copies of libpython in plpython.so and hstore_plpython.so, and those can't interoperate on the same data. There are some other infelicities like assuming that nobody ever installs a private version of Python on a macOS machine. Hence, forget about looking in $python_configdir for the Python shlib; as far as I can tell no version of Python has ever put one there, and certainly no currently-supported version does. Also, rather than relying on get_config_vars('SO'), just try all the possibilities for shlib extensions. Also, rather than trusting Py_ENABLE_SHARED, believe we've found a shlib only if it has a recognized extension. Last, explicitly cope with the possibility that the shlib is really in /usr/lib and $python_libdir is a red herring --- this is the actual situation on older macOS, but we were only accidentally working with it. Discussion: <5300.1475592228@sss.pgh.pa.us> --- config/python.m4 | 57 ++++++++++++++++++++++++++----------- configure | 73 ++++++++++++++++++------------------------------ configure.in | 34 ---------------------- 3 files changed, 67 insertions(+), 97 deletions(-) diff --git a/config/python.m4 b/config/python.m4 index b95c8ed3b3..108d52cb07 100644 --- a/config/python.m4 +++ b/config/python.m4 @@ -58,36 +58,59 @@ AC_SUBST(python_includespec)[]dnl # PGAC_CHECK_PYTHON_EMBED_SETUP # ----------------------------- # -# Note: selecting libpython from python_configdir works in all Python -# releases, but it generally finds a non-shared library, which means -# that we are binding the python interpreter right into libplpython.so. -# In Python 2.3 and up there should be a shared library available in -# the main library location. +# Set python_libdir to the path of the directory containing the Python shared +# library. Set python_libspec to the -L/-l linker switches needed to link it. +# Set python_additional_libs to contain any additional linker switches needed +# for subsidiary libraries. +# +# In modern, well-configured Python installations, LIBDIR gives the correct +# directory name and LDLIBRARY is the file name of the shlib. But in older +# installations LDLIBRARY is frequently a useless path fragment, and it's also +# possible that the shlib is in a standard library directory such as /usr/lib +# so that LIBDIR is of no interest. We must also check that what we found is +# a shared library not a plain library, which we do by checking its extension. +# (We used to rely on Py_ENABLE_SHARED, but that only tells us that a shlib +# exists, not that we found it.) AC_DEFUN([PGAC_CHECK_PYTHON_EMBED_SETUP], [AC_REQUIRE([_PGAC_CHECK_PYTHON_DIRS]) AC_MSG_CHECKING([how to link an embedded Python application]) python_libdir=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBDIR'))))"` python_ldlibrary=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDLIBRARY'))))"` -python_so=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('SO'))))"` -ldlibrary=`echo "${python_ldlibrary}" | sed "s/${python_so}$//"` -python_enable_shared=`${PYTHON} -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_vars().get('Py_ENABLE_SHARED',0))"` -if test x"${python_libdir}" != x"" -a x"${python_ldlibrary}" != x"" -a x"${python_ldlibrary}" != x"${ldlibrary}" +# If LDLIBRARY exists and has a shlib extension, use it verbatim. +ldlibrary=`echo "${python_ldlibrary}" | sed -e 's/\.so$//' -e 's/\.dll$//' -e 's/\.dylib$//' -e 's/\.sl$//'` +if test -e "${python_libdir}/${python_ldlibrary}" -a x"${python_ldlibrary}" != x"${ldlibrary}" then - # New way: use the official shared library ldlibrary=`echo "${ldlibrary}" | sed "s/^lib//"` - python_libspec="-L${python_libdir} -l${ldlibrary}" else - # Old way: use libpython from python_configdir - python_libdir="${python_configdir}" - # LDVERSION was introduced in Python 3.2. + # Otherwise, guess the base name of the shlib. + # LDVERSION was added in Python 3.2, before that use $python_version. python_ldversion=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDVERSION'))))"` - if test x"${python_ldversion}" = x""; then - python_ldversion=$python_version + if test x"${python_ldversion}" != x""; then + ldlibrary="python${python_ldversion}" + else + ldlibrary="python${python_version}" + fi + # Search for a likely-looking file. + found_shlib=0 + for d in "${python_libdir}" /usr/lib64 /usr/lib; do + for e in .so .dll .dylib .sl; do + if test -e "$d/lib${ldlibrary}$e"; then + python_libdir="$d" + found_shlib=1 + break 2 + fi + done + done + if test "$found_shlib" != 1; then + AC_MSG_ERROR([could not find shared library for Python +You might have to rebuild your Python installation. Refer to the +documentation for details. Use --without-python to disable building +PL/Python.]) fi - python_libspec="-L${python_libdir} -lpython${python_ldversion}" fi +python_libspec="-L${python_libdir} -l${ldlibrary}" python_additional_libs=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBS','LIBC','LIBM','BASEMODLIBS'))))"` diff --git a/configure b/configure index 1021fd539e..7bb8f8b1c6 100755 --- a/configure +++ b/configure @@ -7622,25 +7622,40 @@ $as_echo_n "checking how to link an embedded Python application... " >&6; } python_libdir=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBDIR'))))"` python_ldlibrary=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDLIBRARY'))))"` -python_so=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('SO'))))"` -ldlibrary=`echo "${python_ldlibrary}" | sed "s/${python_so}$//"` -python_enable_shared=`${PYTHON} -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_vars().get('Py_ENABLE_SHARED',0))"` -if test x"${python_libdir}" != x"" -a x"${python_ldlibrary}" != x"" -a x"${python_ldlibrary}" != x"${ldlibrary}" +# If LDLIBRARY exists and has a shlib extension, use it verbatim. +ldlibrary=`echo "${python_ldlibrary}" | sed -e 's/\.so$//' -e 's/\.dll$//' -e 's/\.dylib$//' -e 's/\.sl$//'` +if test -e "${python_libdir}/${python_ldlibrary}" -a x"${python_ldlibrary}" != x"${ldlibrary}" then - # New way: use the official shared library ldlibrary=`echo "${ldlibrary}" | sed "s/^lib//"` - python_libspec="-L${python_libdir} -l${ldlibrary}" else - # Old way: use libpython from python_configdir - python_libdir="${python_configdir}" - # LDVERSION was introduced in Python 3.2. + # Otherwise, guess the base name of the shlib. + # LDVERSION was added in Python 3.2, before that use $python_version. python_ldversion=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDVERSION'))))"` - if test x"${python_ldversion}" = x""; then - python_ldversion=$python_version + if test x"${python_ldversion}" != x""; then + ldlibrary="python${python_ldversion}" + else + ldlibrary="python${python_version}" + fi + # Search for a likely-looking file. + found_shlib=0 + for d in "${python_libdir}" /usr/lib64 /usr/lib; do + for e in .so .dll .dylib .sl; do + if test -e "$d/lib${ldlibrary}$e"; then + python_libdir="$d" + found_shlib=1 + break 2 + fi + done + done + if test "$found_shlib" != 1; then + as_fn_error $? "could not find shared library for Python +You might have to rebuild your Python installation. Refer to the +documentation for details. Use --without-python to disable building +PL/Python." "$LINENO" 5 fi - python_libspec="-L${python_libdir} -lpython${python_ldversion}" fi +python_libspec="-L${python_libdir} -l${ldlibrary}" python_additional_libs=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBS','LIBC','LIBM','BASEMODLIBS'))))"` @@ -7649,40 +7664,6 @@ $as_echo "${python_libspec} ${python_additional_libs}" >&6; } - - # We need libpython as a shared library. With Python >=2.5, we - # check the Py_ENABLE_SHARED setting. On Debian, the setting is not - # correct before the jessie release (http://bugs.debian.org/695979). - # We also want to support older Python versions. So as a fallback - # we see if there is a file that is named like a shared library. - - if test "$python_enable_shared" != 1; then - if test "$PORTNAME" = darwin; then - # macOS does supply a .dylib even though Py_ENABLE_SHARED does - # not get set. The file detection logic below doesn't succeed - # on older macOS versions, so make it explicit. - python_enable_shared=1 - elif test "$PORTNAME" = win32; then - # Windows also needs an explicit override. - python_enable_shared=1 - else - # We don't know the platform shared library extension here yet, - # so we try some candidates. - for dlsuffix in .so .sl; do - if ls "$python_libdir"/libpython*${dlsuffix}* >/dev/null 2>&1; then - python_enable_shared=1 - break - fi - done - fi - fi - - if test "$python_enable_shared" != 1; then - as_fn_error $? "cannot build PL/Python because libpython is not a shared library -You might have to rebuild your Python installation. Refer to the -documentation for details. Use --without-python to disable building -PL/Python." "$LINENO" 5 - fi fi if test "$cross_compiling" = yes && test -z "$with_system_tzdata"; then diff --git a/configure.in b/configure.in index 9850d993ff..40f3c093f1 100644 --- a/configure.in +++ b/configure.in @@ -934,40 +934,6 @@ fi if test "$with_python" = yes; then PGAC_PATH_PYTHON PGAC_CHECK_PYTHON_EMBED_SETUP - - # We need libpython as a shared library. With Python >=2.5, we - # check the Py_ENABLE_SHARED setting. On Debian, the setting is not - # correct before the jessie release (http://bugs.debian.org/695979). - # We also want to support older Python versions. So as a fallback - # we see if there is a file that is named like a shared library. - - if test "$python_enable_shared" != 1; then - if test "$PORTNAME" = darwin; then - # macOS does supply a .dylib even though Py_ENABLE_SHARED does - # not get set. The file detection logic below doesn't succeed - # on older macOS versions, so make it explicit. - python_enable_shared=1 - elif test "$PORTNAME" = win32; then - # Windows also needs an explicit override. - python_enable_shared=1 - else - # We don't know the platform shared library extension here yet, - # so we try some candidates. - for dlsuffix in .so .sl; do - if ls "$python_libdir"/libpython*${dlsuffix}* >/dev/null 2>&1; then - python_enable_shared=1 - break - fi - done - fi - fi - - if test "$python_enable_shared" != 1; then - AC_MSG_ERROR([cannot build PL/Python because libpython is not a shared library -You might have to rebuild your Python installation. Refer to the -documentation for details. Use --without-python to disable building -PL/Python.]) - fi fi if test "$cross_compiling" = yes && test -z "$with_system_tzdata"; then From fc76259f5b8473dbd3d2009b0e4a267cf3a7e704 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 4 Oct 2016 16:38:45 -0400 Subject: [PATCH 277/871] Huh, we do need to look in $python_configdir for the Python shlib. Debian does it that way, for no doubt what seems to them a good reason. Thanks to Aidan Van Dyk for confirmation. --- config/python.m4 | 13 ++++++++----- configure | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/config/python.m4 b/config/python.m4 index 108d52cb07..20f5e46a32 100644 --- a/config/python.m4 +++ b/config/python.m4 @@ -67,10 +67,12 @@ AC_SUBST(python_includespec)[]dnl # directory name and LDLIBRARY is the file name of the shlib. But in older # installations LDLIBRARY is frequently a useless path fragment, and it's also # possible that the shlib is in a standard library directory such as /usr/lib -# so that LIBDIR is of no interest. We must also check that what we found is -# a shared library not a plain library, which we do by checking its extension. -# (We used to rely on Py_ENABLE_SHARED, but that only tells us that a shlib -# exists, not that we found it.) +# so that LIBDIR is irrelevant. Also, some packagers put the .so symlink for +# the shlib in ${python_configdir} even though Python itself never does. +# We must also check that what we found is a shared library not a plain +# library, which we do by checking its extension. (We used to rely on +# Py_ENABLE_SHARED, but that only tells us that a shlib exists, not that +# we found it.) AC_DEFUN([PGAC_CHECK_PYTHON_EMBED_SETUP], [AC_REQUIRE([_PGAC_CHECK_PYTHON_DIRS]) AC_MSG_CHECKING([how to link an embedded Python application]) @@ -94,7 +96,8 @@ else fi # Search for a likely-looking file. found_shlib=0 - for d in "${python_libdir}" /usr/lib64 /usr/lib; do + for d in "${python_libdir}" "${python_configdir}" /usr/lib64 /usr/lib + do for e in .so .dll .dylib .sl; do if test -e "$d/lib${ldlibrary}$e"; then python_libdir="$d" diff --git a/configure b/configure index 7bb8f8b1c6..ef3526853c 100755 --- a/configure +++ b/configure @@ -7639,7 +7639,8 @@ else fi # Search for a likely-looking file. found_shlib=0 - for d in "${python_libdir}" /usr/lib64 /usr/lib; do + for d in "${python_libdir}" "${python_configdir}" /usr/lib64 /usr/lib + do for e in .so .dll .dylib .sl; do if test -e "$d/lib${ldlibrary}$e"; then python_libdir="$d" From eda04886c1e048d695728206504ab4198462168e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 4 Oct 2016 17:49:07 -0400 Subject: [PATCH 278/871] Avoid direct cross-module links in hstore_plperl and ltree_plpython, too. Just turning the crank on the project started in commit d51924be8. These cases turn out to be exact subsets of the boilerplate needed for hstore_plpython. Discussion: <2652.1475512158@sss.pgh.pa.us> --- contrib/hstore_plperl/Makefile | 16 +++--- contrib/hstore_plperl/hstore_plperl--1.0.sql | 5 -- contrib/hstore_plperl/hstore_plperl.c | 54 +++++++++++++++++++ contrib/hstore_plperl/hstore_plperlu--1.0.sql | 5 -- contrib/ltree_plpython/Makefile | 19 ++++--- contrib/ltree_plpython/ltree_plpython.c | 29 ++++++++++ .../ltree_plpython/ltree_plpython2u--1.0.sql | 5 -- .../ltree_plpython/ltree_plpython3u--1.0.sql | 5 -- .../ltree_plpython/ltree_plpythonu--1.0.sql | 5 -- src/tools/msvc/Mkvcbuild.pm | 3 +- 10 files changed, 102 insertions(+), 44 deletions(-) diff --git a/contrib/hstore_plperl/Makefile b/contrib/hstore_plperl/Makefile index b3b8654bc8..41d34357f9 100644 --- a/contrib/hstore_plperl/Makefile +++ b/contrib/hstore_plperl/Makefile @@ -23,20 +23,20 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -# In configurations that forbid undefined symbols in libraries, link with each -# dependency. This does preclude pgxs builds. +# We must link libperl explicitly ifeq ($(PORTNAME), aix) rpathdir = $(pkglibdir):$(perl_archlibexp)/CORE -SHLIB_LINK += ../hstore/libhstore.exp $(perl_embed_ldflags) -endif +SHLIB_LINK += $(perl_embed_ldflags) +else ifeq ($(PORTNAME), win32) # these settings are the same as for plperl override CPPFLAGS += -DPLPERL_HAVE_UID_GID -Wno-comment -SHLIB_LINK += ../hstore/libhstore.a $(sort $(wildcard ../../src/pl/plperl/libperl*.a)) +# ... see silliness in plperl Makefile ... +SHLIB_LINK += $(sort $(wildcard ../../src/pl/plperl/libperl*.a)) +else +rpathdir = $(perl_archlibexp)/CORE +SHLIB_LINK += $(perl_embed_ldflags) endif - -ifeq ($(PORTNAME), cygwin) -SHLIB_LINK += -L../hstore -l hstore $(perl_embed_ldflags) endif # As with plperl we need to make sure that the CORE directory is included diff --git a/contrib/hstore_plperl/hstore_plperl--1.0.sql b/contrib/hstore_plperl/hstore_plperl--1.0.sql index 9a64fcb18b..af743c8733 100644 --- a/contrib/hstore_plperl/hstore_plperl--1.0.sql +++ b/contrib/hstore_plperl/hstore_plperl--1.0.sql @@ -3,11 +3,6 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION hstore_plperl" to load this file. \quit --- make sure the prerequisite libraries are loaded -LOAD 'plperl'; -SELECT NULL::hstore; - - CREATE FUNCTION hstore_to_plperl(val internal) RETURNS internal LANGUAGE C STRICT IMMUTABLE AS 'MODULE_PATHNAME'; diff --git a/contrib/hstore_plperl/hstore_plperl.c b/contrib/hstore_plperl/hstore_plperl.c index d40a792730..480212f341 100644 --- a/contrib/hstore_plperl/hstore_plperl.c +++ b/contrib/hstore_plperl/hstore_plperl.c @@ -1,5 +1,7 @@ #include "postgres.h" + #undef _ + #include "fmgr.h" #include "plperl.h" #include "plperl_helpers.h" @@ -7,6 +9,58 @@ PG_MODULE_MAGIC; +extern void _PG_init(void); + +/* Linkage to functions in hstore module */ +typedef HStore *(*hstoreUpgrade_t) (Datum orig); +static hstoreUpgrade_t hstoreUpgrade_p; +typedef int (*hstoreUniquePairs_t) (Pairs *a, int32 l, int32 *buflen); +static hstoreUniquePairs_t hstoreUniquePairs_p; +typedef HStore *(*hstorePairs_t) (Pairs *pairs, int32 pcount, int32 buflen); +static hstorePairs_t hstorePairs_p; +typedef size_t (*hstoreCheckKeyLen_t) (size_t len); +static hstoreCheckKeyLen_t hstoreCheckKeyLen_p; +typedef size_t (*hstoreCheckValLen_t) (size_t len); +static hstoreCheckValLen_t hstoreCheckValLen_p; + + +/* + * Module initialize function: fetch function pointers for cross-module calls. + */ +void +_PG_init(void) +{ + /* Asserts verify that typedefs above match original declarations */ + AssertVariableIsOfType(&hstoreUpgrade, hstoreUpgrade_t); + hstoreUpgrade_p = (hstoreUpgrade_t) + load_external_function("$libdir/hstore", "hstoreUpgrade", + true, NULL); + AssertVariableIsOfType(&hstoreUniquePairs, hstoreUniquePairs_t); + hstoreUniquePairs_p = (hstoreUniquePairs_t) + load_external_function("$libdir/hstore", "hstoreUniquePairs", + true, NULL); + AssertVariableIsOfType(&hstorePairs, hstorePairs_t); + hstorePairs_p = (hstorePairs_t) + load_external_function("$libdir/hstore", "hstorePairs", + true, NULL); + AssertVariableIsOfType(&hstoreCheckKeyLen, hstoreCheckKeyLen_t); + hstoreCheckKeyLen_p = (hstoreCheckKeyLen_t) + load_external_function("$libdir/hstore", "hstoreCheckKeyLen", + true, NULL); + AssertVariableIsOfType(&hstoreCheckValLen, hstoreCheckValLen_t); + hstoreCheckValLen_p = (hstoreCheckValLen_t) + load_external_function("$libdir/hstore", "hstoreCheckValLen", + true, NULL); +} + + +/* These defines must be after the module init function */ +#define hstoreUpgrade hstoreUpgrade_p +#define hstoreUniquePairs hstoreUniquePairs_p +#define hstorePairs hstorePairs_p +#define hstoreCheckKeyLen hstoreCheckKeyLen_p +#define hstoreCheckValLen hstoreCheckValLen_p + PG_FUNCTION_INFO_V1(hstore_to_plperl); diff --git a/contrib/hstore_plperl/hstore_plperlu--1.0.sql b/contrib/hstore_plperl/hstore_plperlu--1.0.sql index f355284907..7c3bc86eba 100644 --- a/contrib/hstore_plperl/hstore_plperlu--1.0.sql +++ b/contrib/hstore_plperl/hstore_plperlu--1.0.sql @@ -3,11 +3,6 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION hstore_plperlu" to load this file. \quit --- make sure the prerequisite libraries are loaded -LOAD 'plperl'; -SELECT NULL::hstore; - - CREATE FUNCTION hstore_to_plperlu(val internal) RETURNS internal LANGUAGE C STRICT IMMUTABLE AS 'MODULE_PATHNAME', 'hstore_to_plperl'; diff --git a/contrib/ltree_plpython/Makefile b/contrib/ltree_plpython/Makefile index 08186f19a1..c45b7c2b09 100644 --- a/contrib/ltree_plpython/Makefile +++ b/contrib/ltree_plpython/Makefile @@ -4,7 +4,7 @@ MODULE_big = ltree_plpython$(python_majorversion) OBJS = ltree_plpython.o $(WIN32RES) PGFILEDESC = "ltree_plpython - ltree transform for plpython" -PG_CPPFLAGS = -I$(top_srcdir)/src/pl/plpython $(python_includespec) -I$(top_srcdir)/contrib/ltree +PG_CPPFLAGS = -I$(top_srcdir)/src/pl/plpython $(python_includespec) -I$(top_srcdir)/contrib/ltree -DPLPYTHON_LIBNAME='"plpython$(python_majorversion)"' EXTENSION = ltree_plpythonu ltree_plpython2u ltree_plpython3u DATA = ltree_plpythonu--1.0.sql ltree_plpython2u--1.0.sql ltree_plpython3u--1.0.sql @@ -23,19 +23,18 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -# In configurations that forbid undefined symbols in libraries, link with each -# dependency. This does preclude pgxs builds. +# We must link libpython explicitly ifeq ($(PORTNAME), aix) rpathdir = $(pkglibdir):$(python_libdir) -SHLIB_LINK += $(python_libspec) $(python_additional_libs) $(sort $(wildcard ../../src/pl/plpython/libplpython*.exp)) -endif +SHLIB_LINK += $(python_libspec) $(python_additional_libs) +else ifeq ($(PORTNAME), win32) -SHLIB_LINK += $(sort $(wildcard ../../src/pl/plpython/libpython*.a)) $(sort $(wildcard ../../src/pl/plpython/libplpython*.a)) +# ... see silliness in plpython Makefile ... +SHLIB_LINK += $(sort $(wildcard ../../src/pl/plpython/libpython*.a)) +else +rpathdir = $(python_libdir) +SHLIB_LINK += $(python_libspec) endif - -ifeq ($(PORTNAME), cygwin) -SHLIB_LINK += -L../ltree -lltree -L../../src/pl/plpython \ - -lplpython$(python_majorversion) $(python_libspec) endif REGRESS_OPTS += --load-extension=ltree diff --git a/contrib/ltree_plpython/ltree_plpython.c b/contrib/ltree_plpython/ltree_plpython.c index 26b7b3c275..bdd462a91b 100644 --- a/contrib/ltree_plpython/ltree_plpython.c +++ b/contrib/ltree_plpython/ltree_plpython.c @@ -1,10 +1,39 @@ #include "postgres.h" + #include "fmgr.h" #include "plpython.h" #include "ltree.h" PG_MODULE_MAGIC; +extern void _PG_init(void); + +/* Linkage to functions in plpython module */ +#if PY_MAJOR_VERSION >= 3 +typedef PyObject *(*PLyUnicode_FromStringAndSize_t) (const char *s, Py_ssize_t size); +static PLyUnicode_FromStringAndSize_t PLyUnicode_FromStringAndSize_p; +#endif + + +/* + * Module initialize function: fetch function pointers for cross-module calls. + */ +void +_PG_init(void) +{ + /* Asserts verify that typedefs above match original declarations */ +#if PY_MAJOR_VERSION >= 3 + AssertVariableIsOfType(&PLyUnicode_FromStringAndSize, PLyUnicode_FromStringAndSize_t); + PLyUnicode_FromStringAndSize_p = (PLyUnicode_FromStringAndSize_t) + load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLyUnicode_FromStringAndSize", + true, NULL); +#endif +} + + +/* These defines must be after the module init function */ +#define PLyUnicode_FromStringAndSize PLyUnicode_FromStringAndSize_p + PG_FUNCTION_INFO_V1(ltree_to_plpython); diff --git a/contrib/ltree_plpython/ltree_plpython2u--1.0.sql b/contrib/ltree_plpython/ltree_plpython2u--1.0.sql index 62531371bf..5c4a703701 100644 --- a/contrib/ltree_plpython/ltree_plpython2u--1.0.sql +++ b/contrib/ltree_plpython/ltree_plpython2u--1.0.sql @@ -3,11 +3,6 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION ltree_plpython2u" to load this file. \quit --- make sure the prerequisite libraries are loaded -LOAD 'plpython2'; -SELECT NULL::ltree; - - CREATE FUNCTION ltree_to_plpython2(val internal) RETURNS internal LANGUAGE C STRICT IMMUTABLE AS 'MODULE_PATHNAME', 'ltree_to_plpython'; diff --git a/contrib/ltree_plpython/ltree_plpython3u--1.0.sql b/contrib/ltree_plpython/ltree_plpython3u--1.0.sql index 3f21d1b721..09ada3c7e8 100644 --- a/contrib/ltree_plpython/ltree_plpython3u--1.0.sql +++ b/contrib/ltree_plpython/ltree_plpython3u--1.0.sql @@ -3,11 +3,6 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION ltree_plpython3u" to load this file. \quit --- make sure the prerequisite libraries are loaded -LOAD 'plpython3'; -SELECT NULL::ltree; - - CREATE FUNCTION ltree_to_plpython3(val internal) RETURNS internal LANGUAGE C STRICT IMMUTABLE AS 'MODULE_PATHNAME', 'ltree_to_plpython'; diff --git a/contrib/ltree_plpython/ltree_plpythonu--1.0.sql b/contrib/ltree_plpython/ltree_plpythonu--1.0.sql index e8deadc62d..ee93edf28b 100644 --- a/contrib/ltree_plpython/ltree_plpythonu--1.0.sql +++ b/contrib/ltree_plpython/ltree_plpythonu--1.0.sql @@ -3,11 +3,6 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION ltree_plpythonu" to load this file. \quit --- make sure the prerequisite libraries are loaded -LOAD 'plpython2'; -- change to plpython3 if that ever becomes the default -SELECT NULL::ltree; - - CREATE FUNCTION ltree_to_plpython(val internal) RETURNS internal LANGUAGE C STRICT IMMUTABLE AS 'MODULE_PATHNAME'; diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index d2ab9e466e..de764dd4d4 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -480,10 +480,11 @@ sub mkvcbuild 'plpython' . $pymajorver, 'src/pl/plpython', 'hstore', 'contrib/hstore'); $hstore_plpython->AddDefine('PLPYTHON_LIBNAME="plpython' . $pymajorver . '"'); - AddTransformModule( + my $ltree_plpython = AddTransformModule( 'ltree_plpython' . $pymajorver, 'contrib/ltree_plpython', 'plpython' . $pymajorver, 'src/pl/plpython', 'ltree', 'contrib/ltree'); + $ltree_plpython->AddDefine('PLPYTHON_LIBNAME="plpython' . $pymajorver . '"'); } if ($solution->{options}->{perl}) From d2ce38e204583bc444eebffd800d492cf56e3b38 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 5 Oct 2016 08:04:52 -0400 Subject: [PATCH 279/871] Rename WAIT_* constants to PG_WAIT_*. Windows apparently has a constant named WAIT_TIMEOUT, and some of these other names are pretty generic, too. Insert "PG_" at the front of each name in order to disambiguate. Michael Paquier --- contrib/postgres_fdw/connection.c | 2 +- src/backend/postmaster/pgstat.c | 36 ++++++++++++------------ src/backend/storage/buffer/bufmgr.c | 2 +- src/backend/storage/ipc/standby.c | 4 +-- src/backend/storage/lmgr/lwlock.c | 8 +++--- src/backend/storage/lmgr/proc.c | 2 +- src/include/pgstat.h | 26 ++++++++--------- src/test/modules/test_shm_mq/setup.c | 2 +- src/test/modules/test_shm_mq/test.c | 2 +- src/test/modules/worker_spi/worker_spi.c | 2 +- 10 files changed, 43 insertions(+), 43 deletions(-) diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c index 9badfe6a7d..bcdddc2db5 100644 --- a/contrib/postgres_fdw/connection.c +++ b/contrib/postgres_fdw/connection.c @@ -497,7 +497,7 @@ pgfdw_get_result(PGconn *conn, const char *query) wc = WaitLatchOrSocket(MyLatch, WL_LATCH_SET | WL_SOCKET_READABLE, PQsocket(conn), - -1L, WAIT_EXTENSION); + -1L, PG_WAIT_EXTENSION); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 5c6cb6b266..a9efee8304 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -3147,31 +3147,31 @@ pgstat_get_wait_event_type(uint32 wait_event_info) switch (classId) { - case WAIT_LWLOCK_NAMED: + case PG_WAIT_LWLOCK_NAMED: event_type = "LWLockNamed"; break; - case WAIT_LWLOCK_TRANCHE: + case PG_WAIT_LWLOCK_TRANCHE: event_type = "LWLockTranche"; break; - case WAIT_LOCK: + case PG_WAIT_LOCK: event_type = "Lock"; break; - case WAIT_BUFFER_PIN: + case PG_WAIT_BUFFER_PIN: event_type = "BufferPin"; break; - case WAIT_ACTIVITY: + case PG_WAIT_ACTIVITY: event_type = "Activity"; break; - case WAIT_CLIENT: + case PG_WAIT_CLIENT: event_type = "Client"; break; - case WAIT_EXTENSION: + case PG_WAIT_EXTENSION: event_type = "Extension"; break; - case WAIT_IPC: + case PG_WAIT_IPC: event_type = "IPC"; break; - case WAIT_TIMEOUT: + case PG_WAIT_TIMEOUT: event_type = "Timeout"; break; default: @@ -3204,41 +3204,41 @@ pgstat_get_wait_event(uint32 wait_event_info) switch (classId) { - case WAIT_LWLOCK_NAMED: - case WAIT_LWLOCK_TRANCHE: + case PG_WAIT_LWLOCK_NAMED: + case PG_WAIT_LWLOCK_TRANCHE: event_name = GetLWLockIdentifier(classId, eventId); break; - case WAIT_LOCK: + case PG_WAIT_LOCK: event_name = GetLockNameFromTagType(eventId); break; - case WAIT_BUFFER_PIN: + case PG_WAIT_BUFFER_PIN: event_name = "BufferPin"; break; - case WAIT_ACTIVITY: + case PG_WAIT_ACTIVITY: { WaitEventActivity w = (WaitEventActivity) wait_event_info; event_name = pgstat_get_wait_activity(w); break; } - case WAIT_CLIENT: + case PG_WAIT_CLIENT: { WaitEventClient w = (WaitEventClient) wait_event_info; event_name = pgstat_get_wait_client(w); break; } - case WAIT_EXTENSION: + case PG_WAIT_EXTENSION: event_name = "Extension"; break; - case WAIT_IPC: + case PG_WAIT_IPC: { WaitEventIPC w = (WaitEventIPC) wait_event_info; event_name = pgstat_get_wait_ipc(w); break; } - case WAIT_TIMEOUT: + case PG_WAIT_TIMEOUT: { WaitEventTimeout w = (WaitEventTimeout) wait_event_info; diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 91dc24c301..2b63cd3ef1 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -3646,7 +3646,7 @@ LockBufferForCleanup(Buffer buffer) SetStartupBufferPinWaitBufId(-1); } else - ProcWaitForSignal(WAIT_BUFFER_PIN); + ProcWaitForSignal(PG_WAIT_BUFFER_PIN); /* * Remove flag marking us as waiter. Normally this will not be set diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c index fb887b3230..875dcec1eb 100644 --- a/src/backend/storage/ipc/standby.c +++ b/src/backend/storage/ipc/standby.c @@ -390,7 +390,7 @@ ResolveRecoveryConflictWithLock(LOCKTAG locktag) } /* Wait to be signaled by the release of the Relation Lock */ - ProcWaitForSignal(WAIT_LOCK | locktag.locktag_type); + ProcWaitForSignal(PG_WAIT_LOCK | locktag.locktag_type); /* * Clear any timeout requests established above. We assume here that the @@ -470,7 +470,7 @@ ResolveRecoveryConflictWithBufferPin(void) } /* Wait to be signaled by UnpinBuffer() */ - ProcWaitForSignal(WAIT_BUFFER_PIN); + ProcWaitForSignal(PG_WAIT_BUFFER_PIN); /* * Clear any timeout requests established above. We assume here that the diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index a90b54ac86..9c6862f41e 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -732,9 +732,9 @@ LWLockReportWaitStart(LWLock *lock) int lockId = T_ID(lock); if (lock->tranche == 0) - pgstat_report_wait_start(WAIT_LWLOCK_NAMED | (uint16) lockId); + pgstat_report_wait_start(PG_WAIT_LWLOCK_NAMED | (uint16) lockId); else - pgstat_report_wait_start(WAIT_LWLOCK_TRANCHE | lock->tranche); + pgstat_report_wait_start(PG_WAIT_LWLOCK_TRANCHE | lock->tranche); } /* @@ -752,10 +752,10 @@ LWLockReportWaitEnd(void) const char * GetLWLockIdentifier(uint32 classId, uint16 eventId) { - if (classId == WAIT_LWLOCK_NAMED) + if (classId == PG_WAIT_LWLOCK_NAMED) return MainLWLockNames[eventId]; - Assert(classId == WAIT_LWLOCK_TRANCHE); + Assert(classId == PG_WAIT_LWLOCK_TRANCHE); /* * It is quite possible that user has registered tranche in one of the diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index dd76094bcd..b2016312a5 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -1214,7 +1214,7 @@ ProcSleep(LOCALLOCK *locallock, LockMethod lockMethodTable) else { WaitLatch(MyLatch, WL_LATCH_SET, 0, - WAIT_LOCK | locallock->tag.lock.locktag_type); + PG_WAIT_LOCK | locallock->tag.lock.locktag_type); ResetLatch(MyLatch); /* check for deadlocks first, as that's probably log-worthy */ if (got_deadlock_timeout) diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 27be5493fd..1c9bf1359e 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -715,15 +715,15 @@ typedef enum BackendState * Wait Classes * ---------- */ -#define WAIT_LWLOCK_NAMED 0x01000000U -#define WAIT_LWLOCK_TRANCHE 0x02000000U -#define WAIT_LOCK 0x03000000U -#define WAIT_BUFFER_PIN 0x04000000U -#define WAIT_ACTIVITY 0x05000000U -#define WAIT_CLIENT 0x06000000U -#define WAIT_EXTENSION 0x07000000U -#define WAIT_IPC 0x08000000U -#define WAIT_TIMEOUT 0x09000000U +#define PG_WAIT_LWLOCK_NAMED 0x01000000U +#define PG_WAIT_LWLOCK_TRANCHE 0x02000000U +#define PG_WAIT_LOCK 0x03000000U +#define PG_WAIT_BUFFER_PIN 0x04000000U +#define PG_WAIT_ACTIVITY 0x05000000U +#define PG_WAIT_CLIENT 0x06000000U +#define PG_WAIT_EXTENSION 0x07000000U +#define PG_WAIT_IPC 0x08000000U +#define PG_WAIT_TIMEOUT 0x09000000U /* ---------- * Wait Events - Activity @@ -735,7 +735,7 @@ typedef enum BackendState */ typedef enum { - WAIT_EVENT_ARCHIVER_MAIN = WAIT_ACTIVITY, + WAIT_EVENT_ARCHIVER_MAIN = PG_WAIT_ACTIVITY, WAIT_EVENT_AUTOVACUUM_MAIN, WAIT_EVENT_BGWRITER_HIBERNATE, WAIT_EVENT_BGWRITER_MAIN, @@ -759,7 +759,7 @@ typedef enum */ typedef enum { - WAIT_EVENT_CLIENT_READ = WAIT_CLIENT, + WAIT_EVENT_CLIENT_READ = PG_WAIT_CLIENT, WAIT_EVENT_CLIENT_WRITE, WAIT_EVENT_SSL_OPEN_SERVER, WAIT_EVENT_WAL_RECEIVER_WAIT_START, @@ -776,7 +776,7 @@ typedef enum */ typedef enum { - WAIT_EVENT_BGWORKER_SHUTDOWN = WAIT_IPC, + WAIT_EVENT_BGWORKER_SHUTDOWN = PG_WAIT_IPC, WAIT_EVENT_BGWORKER_STARTUP, WAIT_EVENT_EXECUTE_GATHER, WAIT_EVENT_MQ_INTERNAL, @@ -796,7 +796,7 @@ typedef enum */ typedef enum { - WAIT_EVENT_BASE_BACKUP_THROTTLE = WAIT_TIMEOUT, + WAIT_EVENT_BASE_BACKUP_THROTTLE = PG_WAIT_TIMEOUT, WAIT_EVENT_PG_SLEEP, WAIT_EVENT_RECOVERY_APPLY_DELAY } WaitEventTimeout; diff --git a/src/test/modules/test_shm_mq/setup.c b/src/test/modules/test_shm_mq/setup.c index cb86f3c37d..5cfffe4471 100644 --- a/src/test/modules/test_shm_mq/setup.c +++ b/src/test/modules/test_shm_mq/setup.c @@ -280,7 +280,7 @@ wait_for_workers_to_become_ready(worker_state *wstate, } /* Wait to be signalled. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EXTENSION); + WaitLatch(MyLatch, WL_LATCH_SET, 0, PG_WAIT_EXTENSION); /* Reset the latch so we don't spin. */ ResetLatch(MyLatch); diff --git a/src/test/modules/test_shm_mq/test.c b/src/test/modules/test_shm_mq/test.c index bf11137a96..b0f3d3daec 100644 --- a/src/test/modules/test_shm_mq/test.c +++ b/src/test/modules/test_shm_mq/test.c @@ -231,7 +231,7 @@ test_shm_mq_pipelined(PG_FUNCTION_ARGS) * have read or written data and therefore there may now be work * for us to do. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EXTENSION); + WaitLatch(MyLatch, WL_LATCH_SET, 0, PG_WAIT_EXTENSION); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } diff --git a/src/test/modules/worker_spi/worker_spi.c b/src/test/modules/worker_spi/worker_spi.c index f8395bfb57..5f495498ae 100644 --- a/src/test/modules/worker_spi/worker_spi.c +++ b/src/test/modules/worker_spi/worker_spi.c @@ -228,7 +228,7 @@ worker_spi_main(Datum main_arg) rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, worker_spi_naptime * 1000L, - WAIT_EXTENSION); + PG_WAIT_EXTENSION); ResetLatch(MyLatch); /* emergency bailout if postmaster has died */ From eb3bc0bd1a68e30f09f36937da5aa5338b3b994c Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 5 Oct 2016 08:24:25 -0400 Subject: [PATCH 280/871] Re-alphabetize #include directives. Thomas Munro --- src/backend/postmaster/bgworker.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 028a9eed2d..3a2929a00b 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -14,11 +14,11 @@ #include -#include "miscadmin.h" #include "libpq/pqsignal.h" +#include "miscadmin.h" +#include "pgstat.h" #include "postmaster/bgworker_internals.h" #include "postmaster/postmaster.h" -#include "pgstat.h" #include "storage/barrier.h" #include "storage/dsm.h" #include "storage/ipc.h" From ddd4f82cb6f65354776541dfac3bedf680e0e303 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 5 Oct 2016 11:44:57 -0400 Subject: [PATCH 281/871] In python shlib probe, cater for OpenBSD, which omits the .so symlink. Most Unix-oid platforms provide a symlink "libfoo.so" -> "libfoo.so.n.n" to allow the linker to resolve a reference "-lfoo" to a version-numbered shared library. OpenBSD has apparently hacked ld(1) to do this internally, because there are no such symlinks to be found in their library directories. Tweak the new code in PGAC_CHECK_PYTHON_EMBED_SETUP to cope. Per buildfarm member curculio. --- config/python.m4 | 22 ++++++++++++++++++++-- configure | 22 ++++++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/config/python.m4 b/config/python.m4 index 20f5e46a32..e29f05987b 100644 --- a/config/python.m4 +++ b/config/python.m4 @@ -85,6 +85,7 @@ ldlibrary=`echo "${python_ldlibrary}" | sed -e 's/\.so$//' -e 's/\.dll$//' -e 's if test -e "${python_libdir}/${python_ldlibrary}" -a x"${python_ldlibrary}" != x"${ldlibrary}" then ldlibrary=`echo "${ldlibrary}" | sed "s/^lib//"` + found_shlib=1 else # Otherwise, guess the base name of the shlib. # LDVERSION was added in Python 3.2, before that use $python_version. @@ -98,6 +99,7 @@ else found_shlib=0 for d in "${python_libdir}" "${python_configdir}" /usr/lib64 /usr/lib do + # We don't know the platform DLSUFFIX here, so check 'em all. for e in .so .dll .dylib .sl; do if test -e "$d/lib${ldlibrary}$e"; then python_libdir="$d" @@ -106,12 +108,28 @@ else fi done done + # Some platforms (OpenBSD) require us to accept a bare versioned shlib + # (".so.n.n") as well. However, check this only after failing to find + # ".so" anywhere, because yet other platforms (Debian) put the .so + # symlink in a different directory from the underlying versioned shlib. if test "$found_shlib" != 1; then - AC_MSG_ERROR([could not find shared library for Python + for d in "${python_libdir}" "${python_configdir}" /usr/lib64 /usr/lib + do + for f in "$d/lib${ldlibrary}.so."* ; do + if test -e "$f"; then + python_libdir="$d" + found_shlib=1 + break 2 + fi + done + done + fi +fi +if test "$found_shlib" != 1; then + AC_MSG_ERROR([could not find shared library for Python You might have to rebuild your Python installation. Refer to the documentation for details. Use --without-python to disable building PL/Python.]) - fi fi python_libspec="-L${python_libdir} -l${ldlibrary}" diff --git a/configure b/configure index ef3526853c..3d08f5a5e9 100755 --- a/configure +++ b/configure @@ -7628,6 +7628,7 @@ ldlibrary=`echo "${python_ldlibrary}" | sed -e 's/\.so$//' -e 's/\.dll$//' -e 's if test -e "${python_libdir}/${python_ldlibrary}" -a x"${python_ldlibrary}" != x"${ldlibrary}" then ldlibrary=`echo "${ldlibrary}" | sed "s/^lib//"` + found_shlib=1 else # Otherwise, guess the base name of the shlib. # LDVERSION was added in Python 3.2, before that use $python_version. @@ -7641,6 +7642,7 @@ else found_shlib=0 for d in "${python_libdir}" "${python_configdir}" /usr/lib64 /usr/lib do + # We don't know the platform DLSUFFIX here, so check 'em all. for e in .so .dll .dylib .sl; do if test -e "$d/lib${ldlibrary}$e"; then python_libdir="$d" @@ -7649,12 +7651,28 @@ else fi done done + # Some platforms (OpenBSD) require us to accept a bare versioned shlib + # (".so.n.n") as well. However, check this only after failing to find + # ".so" anywhere, because yet other platforms (Debian) put the .so + # symlink in a different directory from the underlying versioned shlib. if test "$found_shlib" != 1; then - as_fn_error $? "could not find shared library for Python + for d in "${python_libdir}" "${python_configdir}" /usr/lib64 /usr/lib + do + for f in "$d/lib${ldlibrary}.so."* ; do + if test -e "$f"; then + python_libdir="$d" + found_shlib=1 + break 2 + fi + done + done + fi +fi +if test "$found_shlib" != 1; then + as_fn_error $? "could not find shared library for Python You might have to rebuild your Python installation. Refer to the documentation for details. Use --without-python to disable building PL/Python." "$LINENO" 5 - fi fi python_libspec="-L${python_libdir} -l${ldlibrary}" From 61f9e7ba3c4e4bde887f980b9316fb818ede59b6 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 5 Oct 2016 13:08:48 -0400 Subject: [PATCH 282/871] Update obsolete comments and perldoc. Loose ends from commit 2a0f89cd717ce6d49cdc47850577823682167e87. Daniel Gustafsson --- src/test/perl/PostgresNode.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 4b225e1471..535d6c0e7c 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -1192,7 +1192,7 @@ sub psql =item $node->poll_query_until(dbname, query) Run a query once a second, until it returns 't' (i.e. SQL boolean true). -Continues polling if psql returns an error result. Times out after 90 seconds. +Continues polling if psql returns an error result. Times out after 180 seconds. =cut @@ -1222,7 +1222,7 @@ sub poll_query_until $attempts++; } - # The query result didn't change in 90 seconds. Give up. Print the stderr + # The query result didn't change in 180 seconds. Give up. Print the stderr # from the last attempt, hopefully that's useful for debugging. diag $stderr; return 0; From 11c0e743b67acc9a0406b6058ed24de5e5527cf0 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 5 Oct 2016 22:47:23 -0400 Subject: [PATCH 283/871] Try to fix python shlib probe for MinGW. Per discussion with Andrew Dunstan, it seems there are three peculiarities of the Python installation on MinGW (or at least, of the instance on buildfarm member frogmouth). First, the library name doesn't contain "2.7" but just "27". It looks like we can deal with that by consulting get_config_vars('VERSION') to see whether a dot should be used or not. Second, the library might be in c:/Windows/System32, analogously to it possibly being in /usr/lib on Unix-oid platforms. Third, it's apparently not standard to use the prefix "lib" on the file name. This patch will accept files with or without "lib", but the first part of that may well be dead code. --- config/python.m4 | 25 +++++++++++++++++++++++-- configure | 25 +++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/config/python.m4 b/config/python.m4 index e29f05987b..babf5ab1db 100644 --- a/config/python.m4 +++ b/config/python.m4 @@ -88,12 +88,18 @@ then found_shlib=1 else # Otherwise, guess the base name of the shlib. - # LDVERSION was added in Python 3.2, before that use $python_version. + # LDVERSION was added in Python 3.2, before that use VERSION, + # or failing that, $python_version from _PGAC_CHECK_PYTHON_DIRS. python_ldversion=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDVERSION'))))"` if test x"${python_ldversion}" != x""; then ldlibrary="python${python_ldversion}" else - ldlibrary="python${python_version}" + python_version_var=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('VERSION'))))"` + if test x"${python_version_var}" != x""; then + ldlibrary="python${python_version_var}" + else + ldlibrary="python${python_version}" + fi fi # Search for a likely-looking file. found_shlib=0 @@ -124,6 +130,21 @@ else done done fi + # As usual, Windows has its own ideas. c:/Windows/System32 takes the + # place of /usr/lib as a possible default library location, and the + # "lib" prefix might not be there. + if test "$found_shlib" != 1 -a "$PORTNAME" = win32 ; then + for d in "${python_libdir}" "${python_configdir}" c:/Windows/System32 + do + for f in "$d/lib${ldlibrary}.dll" "$d/${ldlibrary}.dll" ; do + if test -e "$f"; then + python_libdir="$d" + found_shlib=1 + break 2 + fi + done + done + fi fi if test "$found_shlib" != 1; then AC_MSG_ERROR([could not find shared library for Python diff --git a/configure b/configure index 3d08f5a5e9..204daa056d 100755 --- a/configure +++ b/configure @@ -7631,12 +7631,18 @@ then found_shlib=1 else # Otherwise, guess the base name of the shlib. - # LDVERSION was added in Python 3.2, before that use $python_version. + # LDVERSION was added in Python 3.2, before that use VERSION, + # or failing that, $python_version from _PGAC_CHECK_PYTHON_DIRS. python_ldversion=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LDVERSION'))))"` if test x"${python_ldversion}" != x""; then ldlibrary="python${python_ldversion}" else - ldlibrary="python${python_version}" + python_version_var=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('VERSION'))))"` + if test x"${python_version_var}" != x""; then + ldlibrary="python${python_version_var}" + else + ldlibrary="python${python_version}" + fi fi # Search for a likely-looking file. found_shlib=0 @@ -7667,6 +7673,21 @@ else done done fi + # As usual, Windows has its own ideas. c:/Windows/System32 takes the + # place of /usr/lib as a possible default library location, and the + # "lib" prefix might not be there. + if test "$found_shlib" != 1 -a "$PORTNAME" = win32 ; then + for d in "${python_libdir}" "${python_configdir}" c:/Windows/System32 + do + for f in "$d/lib${ldlibrary}.dll" "$d/${ldlibrary}.dll" ; do + if test -e "$f"; then + python_libdir="$d" + found_shlib=1 + break 2 + fi + done + done + fi fi if test "$found_shlib" != 1; then as_fn_error $? "could not find shared library for Python From bfe2e847811a4d0a46c8cdf16195c0e5929b6f83 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 5 Oct 2016 23:03:55 -0400 Subject: [PATCH 284/871] Remove -Wl,-undefined,dynamic_lookup in macOS build. We don't need this anymore, and it prevents build-time error checking that's usually good to have, so remove it. Undoes one change of commit cac765820. Unfortunately, it's much harder to get a similar effect on other common platforms, because we don't want the linker to throw errors for symbols that will be resolved in the core backend. Only macOS and AIX expect the core backend executable to be available while linking loadable modules, so only these platforms can usefully throw errors for unresolved symbols at link time. Discussion: <2652.1475512158@sss.pgh.pa.us> --- src/Makefile.shlib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile.shlib b/src/Makefile.shlib index f64eb4d9c5..87c80c5d01 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -127,7 +127,7 @@ ifeq ($(PORTNAME), darwin) else # loadable module DLSUFFIX = .so - LINK.shared = $(COMPILER) -bundle -multiply_defined suppress -Wl,-undefined,dynamic_lookup + LINK.shared = $(COMPILER) -bundle -multiply_defined suppress endif BUILD.exports = $(AWK) '/^[^\#]/ {printf "_%s\n",$$1}' $< >$@ exports_file = $(SHLIB_EXPORTS:%.txt=%.list) From b56fb691b0033f9b86e0552bd5adfd485f05eef6 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 6 Oct 2016 09:46:40 +0300 Subject: [PATCH 285/871] Fix excessive memory consumption in the new sort pre-reading code. LogicalTapeRewind() should not allocate large read buffer, if the tape is completely empty. The calling code relies on that, for its calculation of how much memory to allocate for the read buffers. That lead to massive overallocation of memory, if maxTapes was high, but only a few tapes were actually used. Reported by Tomas Vondra Discussion: <7303da46-daf7-9c68-3cc1-9f83235cf37e@2ndquadrant.com> --- src/backend/utils/sort/logtape.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index caa6960b95..6cc06b24e0 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -778,11 +778,16 @@ LogicalTapeRewind(LogicalTapeSet *lts, int tapenum, bool forWrite) datablocknum = ltsRewindFrozenIndirectBlock(lts, lt->indirect); } - /* Allocate a read buffer */ + /* Allocate a read buffer (unless the tape is empty) */ if (lt->buffer) pfree(lt->buffer); - lt->buffer = palloc(lt->read_buffer_size); - lt->buffer_size = lt->read_buffer_size; + lt->buffer = NULL; + lt->buffer_size = 0; + if (datablocknum != -1L) + { + lt->buffer = palloc(lt->read_buffer_size); + lt->buffer_size = lt->read_buffer_size; + } /* Read the first block, or reset if tape is empty */ lt->curBlockNumber = 0L; From d7eb76b908a3aac68c23c7d3553c65c80e92b823 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 6 Oct 2016 13:24:46 +0300 Subject: [PATCH 286/871] Disable synchronous commits in pg_rewind. If you point pg_rewind to a server that is using synchronous replication, with "pg_rewind --source-server=...", and the replication is not working for some reason, pg_rewind will get stuck because it creates a temporary table, which needs to be replicated. You could call broken replication a pilot error, but pg_rewind is often used in special circumstances, when there are changes to the replication setup. We don't do any "real" updates, and we don't care about fsyncing or replicating the operations on the temporary tables, so fix that by setting synchronous_commit off. Michael Banck, Michael Paquier. Backpatch to 9.5, where pg_rewind was introduced. Discussion: <20161005143938.GA12247@nighthawk.caipicrew.dd-dns.de> --- src/bin/pg_rewind/libpq_fetch.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index 92390099eb..5128e1c250 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -49,6 +49,7 @@ void libpqConnect(const char *connstr) { char *str; + PGresult *res; conn = PQconnectdb(connstr); if (PQstatus(conn) == CONNECTION_BAD) @@ -77,6 +78,19 @@ libpqConnect(const char *connstr) if (strcmp(str, "on") != 0) pg_fatal("full_page_writes must be enabled in the source server\n"); pg_free(str); + + /* + * Although we don't do any "real" updates, we do work with a temporary + * table. We don't care about synchronous commit for that. It doesn't + * otherwise matter much, but if the server is using synchronous + * replication, and replication isn't working for some reason, we don't + * want to get stuck, waiting for it to start working again. + */ + res = PQexec(conn, "SET synchronous_commit = off"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + pg_fatal("could not set up connection context: %s", + PQresultErrorMessage(res)); + PQclear(res); } /* From 8bb14cdd33deecc6977cf5638f73e80f76ab658b Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 7 Oct 2016 12:20:39 +0300 Subject: [PATCH 287/871] Don't share SSL_CTX between libpq connections. There were several issues with the old coding: 1. There was a race condition, if two threads opened a connection at the same time. We used a mutex around SSL_CTX_* calls, but that was not enough, e.g. if one thread SSL_CTX_load_verify_locations() with one path, and another thread set it with a different path, before the first thread got to establish the connection. 2. Opening two different connections, with different sslrootcert settings, seemed to fail outright with "SSL error: block type is not 01". Not sure why. 3. We created the SSL object, before calling SSL_CTX_load_verify_locations and SSL_CTX_use_certificate_chain_file on the SSL context. That was wrong, because the options set on the SSL context are propagated to the SSL object, when the SSL object is created. If they are set after the SSL object has already been created, they won't take effect until the next connection. (This is bug #14329) At least some of these could've been fixed while still using a shared context, but it would've been more complicated and error-prone. To keep things simple, let's just use a separate SSL context for each connection, and accept the overhead. Backpatch to all supported versions. Report, analysis and test case by Kacper Zuk. Discussion: <20160920101051.1355.79453@wrigleys.postgresql.org> --- src/interfaces/libpq/fe-secure-openssl.c | 397 ++++++++++------------- src/test/ssl/Makefile | 6 +- src/test/ssl/README | 4 + src/test/ssl/ServerSetup.pm | 6 +- src/test/ssl/ssl/client+client_ca.crt | 25 ++ src/test/ssl/t/001_ssltests.pl | 10 +- 6 files changed, 211 insertions(+), 237 deletions(-) create mode 100644 src/test/ssl/ssl/client+client_ca.crt diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index 2dcdce7265..c83c1acae4 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -80,12 +80,7 @@ static int my_SSL_set_fd(PGconn *conn, int fd); static bool pq_init_ssl_lib = true; static bool pq_init_crypto_lib = true; -/* - * SSL_context is currently shared between threads and therefore we need to be - * careful to lock around any usage of it when providing thread safety. - * ssl_config_mutex is the mutex that we use to protect it. - */ -static SSL_CTX *SSL_context = NULL; +static bool ssl_lib_initialized = false; #ifdef ENABLE_THREAD_SAFETY static long ssl_open_connections = 0; @@ -133,44 +128,9 @@ pgtls_open_client(PGconn *conn) /* First time through? */ if (conn->ssl == NULL) { -#ifdef ENABLE_THREAD_SAFETY - int rc; -#endif - -#ifdef ENABLE_THREAD_SAFETY - if ((rc = pthread_mutex_lock(&ssl_config_mutex))) - { - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("could not acquire mutex: %s\n"), strerror(rc)); - return PGRES_POLLING_FAILED; - } -#endif - /* Create a connection-specific SSL object */ - if (!(conn->ssl = SSL_new(SSL_context)) || - !SSL_set_app_data(conn->ssl, conn) || - !my_SSL_set_fd(conn, conn->sock)) - { - char *err = SSLerrmessage(ERR_get_error()); - - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("could not establish SSL connection: %s\n"), - err); - SSLerrfree(err); -#ifdef ENABLE_THREAD_SAFETY - pthread_mutex_unlock(&ssl_config_mutex); -#endif - pgtls_close(conn); - - return PGRES_POLLING_FAILED; - } - conn->ssl_in_use = true; - -#ifdef ENABLE_THREAD_SAFETY - pthread_mutex_unlock(&ssl_config_mutex); -#endif - /* - * Load client certificate, private key, and trusted CA certs. + * Create a connection-specific SSL object, and load client certificate, + * private key, and trusted CA certs. */ if (initialize_SSL(conn) != 0) { @@ -771,8 +731,7 @@ pq_lockingcallback(int mode, int n, const char *file, int line) #endif /* ENABLE_THREAD_SAFETY && HAVE_CRYPTO_LOCK */ /* - * Initialize SSL system, in particular creating the SSL_context object - * that will be shared by all SSL-using connections in this process. + * Initialize SSL library. * * In threadsafe mode, this includes setting up libcrypto callback functions * to do thread locking. @@ -851,7 +810,7 @@ pgtls_init(PGconn *conn) #endif /* HAVE_CRYPTO_LOCK */ #endif /* ENABLE_THREAD_SAFETY */ - if (!SSL_context) + if (!ssl_lib_initialized) { if (pq_init_ssl_lib) { @@ -863,36 +822,7 @@ pgtls_init(PGconn *conn) SSL_load_error_strings(); #endif } - - /* - * We use SSLv23_method() because it can negotiate use of the highest - * mutually supported protocol version, while alternatives like - * TLSv1_2_method() permit only one specific version. Note that we - * don't actually allow SSL v2 or v3, only TLS protocols (see below). - */ - SSL_context = SSL_CTX_new(SSLv23_method()); - if (!SSL_context) - { - char *err = SSLerrmessage(ERR_get_error()); - - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("could not create SSL context: %s\n"), - err); - SSLerrfree(err); -#ifdef ENABLE_THREAD_SAFETY - pthread_mutex_unlock(&ssl_config_mutex); -#endif - return -1; - } - - /* Disable old protocol versions */ - SSL_CTX_set_options(SSL_context, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); - - /* - * Disable OpenSSL's moving-write-buffer sanity check, because it - * causes unnecessary failures in nonblocking send cases. - */ - SSL_CTX_set_mode(SSL_context, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + ssl_lib_initialized = true; } #ifdef ENABLE_THREAD_SAFETY @@ -936,9 +866,8 @@ destroy_ssl_system(void) CRYPTO_set_id_callback(NULL); /* - * We don't free the lock array or the SSL_context. If we get another - * connection in this process, we will just re-use them with the - * existing mutexes. + * We don't free the lock array. If we get another connection in + * this process, we will just re-use them with the existing mutexes. * * This means we leak a little memory on repeated load/unload of the * library. @@ -950,26 +879,22 @@ destroy_ssl_system(void) } /* - * Initialize (potentially) per-connection SSL data, namely the - * client certificate, private key, and trusted CA certs. - * - * conn->ssl must already be created. It receives the connection's client - * certificate and private key. Note however that certificates also get - * loaded into the SSL_context object, and are therefore accessible to all - * connections in this process. This should be OK as long as there aren't - * any hash collisions among the certs. + * Create per-connection SSL object, and load the client certificate, + * private key, and trusted CA certs. * * Returns 0 if OK, -1 on failure (with a message in conn->errorMessage). */ static int initialize_SSL(PGconn *conn) { + SSL_CTX *SSL_context; struct stat buf; char homedir[MAXPGPATH]; char fnbuf[MAXPGPATH]; char sebuf[256]; bool have_homedir; bool have_cert; + bool have_rootcert; EVP_PKEY *pkey = NULL; /* @@ -985,6 +910,123 @@ initialize_SSL(PGconn *conn) else /* won't need it */ have_homedir = false; + /* + * Create a new SSL_CTX object. + * + * We used to share a single SSL_CTX between all connections, but it was + * complicated if connections used different certificates. So now we create + * a separate context for each connection, and accept the overhead. + */ + SSL_context = SSL_CTX_new(SSLv23_method()); + if (!SSL_context) + { + char *err = SSLerrmessage(ERR_get_error()); + + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("could not create SSL context: %s\n"), + err); + SSLerrfree(err); + return -1; + } + + /* Disable old protocol versions */ + SSL_CTX_set_options(SSL_context, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + + /* + * Disable OpenSSL's moving-write-buffer sanity check, because it + * causes unnecessary failures in nonblocking send cases. + */ + SSL_CTX_set_mode(SSL_context, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + + /* + * If the root cert file exists, load it so we can perform certificate + * verification. If sslmode is "verify-full" we will also do further + * verification after the connection has been completed. + */ + if (conn->sslrootcert && strlen(conn->sslrootcert) > 0) + strlcpy(fnbuf, conn->sslrootcert, sizeof(fnbuf)); + else if (have_homedir) + snprintf(fnbuf, sizeof(fnbuf), "%s/%s", homedir, ROOT_CERT_FILE); + else + fnbuf[0] = '\0'; + + if (fnbuf[0] != '\0' && + stat(fnbuf, &buf) == 0) + { + X509_STORE *cvstore; + + if (SSL_CTX_load_verify_locations(SSL_context, fnbuf, NULL) != 1) + { + char *err = SSLerrmessage(ERR_get_error()); + + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("could not read root certificate file \"%s\": %s\n"), + fnbuf, err); + SSLerrfree(err); + SSL_CTX_free(SSL_context); + return -1; + } + + if ((cvstore = SSL_CTX_get_cert_store(SSL_context)) != NULL) + { + if (conn->sslcrl && strlen(conn->sslcrl) > 0) + strlcpy(fnbuf, conn->sslcrl, sizeof(fnbuf)); + else if (have_homedir) + snprintf(fnbuf, sizeof(fnbuf), "%s/%s", homedir, ROOT_CRL_FILE); + else + fnbuf[0] = '\0'; + + /* Set the flags to check against the complete CRL chain */ + if (fnbuf[0] != '\0' && + X509_STORE_load_locations(cvstore, fnbuf, NULL) == 1) + { + /* OpenSSL 0.96 does not support X509_V_FLAG_CRL_CHECK */ +#ifdef X509_V_FLAG_CRL_CHECK + X509_STORE_set_flags(cvstore, + X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL); +#else + char *err = SSLerrmessage(ERR_get_error()); + + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("SSL library does not support CRL certificates (file \"%s\")\n"), + fnbuf); + SSLerrfree(err); + SSL_CTX_free(SSL_context); + return -1; +#endif + } + /* if not found, silently ignore; we do not require CRL */ + } + have_rootcert = true; + } + else + { + /* + * stat() failed; assume root file doesn't exist. If sslmode is + * verify-ca or verify-full, this is an error. Otherwise, continue + * without performing any server cert verification. + */ + if (conn->sslmode[0] == 'v') /* "verify-ca" or "verify-full" */ + { + /* + * The only way to reach here with an empty filename is if + * pqGetHomeDirectory failed. That's a sufficiently unusual case + * that it seems worth having a specialized error message for it. + */ + if (fnbuf[0] == '\0') + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("could not get home directory to locate root certificate file\n" + "Either provide the file or change sslmode to disable server certificate verification.\n")); + else + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("root certificate file \"%s\" does not exist\n" + "Either provide the file or change sslmode to disable server certificate verification.\n"), fnbuf); + SSL_CTX_free(SSL_context); + return -1; + } + have_rootcert = false; + } + /* Read the client certificate file */ if (conn->sslcert && strlen(conn->sslcert) > 0) strlcpy(fnbuf, conn->sslcert, sizeof(fnbuf)); @@ -1010,6 +1052,7 @@ initialize_SSL(PGconn *conn) printfPQExpBuffer(&conn->errorMessage, libpq_gettext("could not open certificate file \"%s\": %s\n"), fnbuf, pqStrerror(errno, sebuf, sizeof(sebuf))); + SSL_CTX_free(SSL_context); return -1; } have_cert = false; @@ -1017,31 +1060,10 @@ initialize_SSL(PGconn *conn) else { /* - * Cert file exists, so load it. Since OpenSSL doesn't provide the - * equivalent of "SSL_use_certificate_chain_file", we actually have to - * load the file twice. The first call loads any extra certs after - * the first one into chain-cert storage associated with the - * SSL_context. The second call loads the first cert (only) into the - * SSL object, where it will be correctly paired with the private key - * we load below. We do it this way so that each connection - * understands which subject cert to present, in case different - * sslcert settings are used for different connections in the same - * process. - * - * NOTE: This function may also modify our SSL_context and therefore - * we have to lock around this call and any places where we use the - * SSL_context struct. + * Cert file exists, so load it. Since OpenSSL doesn't provide the + * equivalent of "SSL_use_certificate_chain_file", we have to load + * it into the SSL context, rather than the SSL object. */ -#ifdef ENABLE_THREAD_SAFETY - int rc; - - if ((rc = pthread_mutex_lock(&ssl_config_mutex))) - { - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("could not acquire mutex: %s\n"), strerror(rc)); - return -1; - } -#endif if (SSL_CTX_use_certificate_chain_file(SSL_context, fnbuf) != 1) { char *err = SSLerrmessage(ERR_get_error()); @@ -1050,34 +1072,42 @@ initialize_SSL(PGconn *conn) libpq_gettext("could not read certificate file \"%s\": %s\n"), fnbuf, err); SSLerrfree(err); - -#ifdef ENABLE_THREAD_SAFETY - pthread_mutex_unlock(&ssl_config_mutex); -#endif - return -1; - } - - if (SSL_use_certificate_file(conn->ssl, fnbuf, SSL_FILETYPE_PEM) != 1) - { - char *err = SSLerrmessage(ERR_get_error()); - - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("could not read certificate file \"%s\": %s\n"), - fnbuf, err); - SSLerrfree(err); -#ifdef ENABLE_THREAD_SAFETY - pthread_mutex_unlock(&ssl_config_mutex); -#endif + SSL_CTX_free(SSL_context); return -1; } /* need to load the associated private key, too */ have_cert = true; + } -#ifdef ENABLE_THREAD_SAFETY - pthread_mutex_unlock(&ssl_config_mutex); -#endif + /* + * The SSL context is now loaded with the correct root and client certificates. + * Create a connection-specific SSL object. The private key is loaded directly + * into the SSL object. (We could load the private key into the context, too, but + * we have done it this way historically, and it doesn't really matter.) + */ + if (!(conn->ssl = SSL_new(SSL_context)) || + !SSL_set_app_data(conn->ssl, conn) || + !my_SSL_set_fd(conn, conn->sock)) + { + char *err = SSLerrmessage(ERR_get_error()); + + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("could not establish SSL connection: %s\n"), + err); + SSLerrfree(err); + SSL_CTX_free(SSL_context); + return -1; } + conn->ssl_in_use = true; + + /* + * SSL contexts are reference counted by OpenSSL. We can free it as soon as we + * have created the SSL object, and it will stick around for as long as it's + * actually needed. + */ + SSL_CTX_free(SSL_context); + SSL_context = NULL; /* * Read the SSL key. If a key is specified, treat it as an engine:key @@ -1236,109 +1266,10 @@ initialize_SSL(PGconn *conn) } /* - * If the root cert file exists, load it so we can perform certificate - * verification. If sslmode is "verify-full" we will also do further - * verification after the connection has been completed. + * If a root cert was loaded, also set our certificate verification callback. */ - if (conn->sslrootcert && strlen(conn->sslrootcert) > 0) - strlcpy(fnbuf, conn->sslrootcert, sizeof(fnbuf)); - else if (have_homedir) - snprintf(fnbuf, sizeof(fnbuf), "%s/%s", homedir, ROOT_CERT_FILE); - else - fnbuf[0] = '\0'; - - if (fnbuf[0] != '\0' && - stat(fnbuf, &buf) == 0) - { - X509_STORE *cvstore; - -#ifdef ENABLE_THREAD_SAFETY - int rc; - - if ((rc = pthread_mutex_lock(&ssl_config_mutex))) - { - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("could not acquire mutex: %s\n"), strerror(rc)); - return -1; - } -#endif - if (SSL_CTX_load_verify_locations(SSL_context, fnbuf, NULL) != 1) - { - char *err = SSLerrmessage(ERR_get_error()); - - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("could not read root certificate file \"%s\": %s\n"), - fnbuf, err); - SSLerrfree(err); -#ifdef ENABLE_THREAD_SAFETY - pthread_mutex_unlock(&ssl_config_mutex); -#endif - return -1; - } - - if ((cvstore = SSL_CTX_get_cert_store(SSL_context)) != NULL) - { - if (conn->sslcrl && strlen(conn->sslcrl) > 0) - strlcpy(fnbuf, conn->sslcrl, sizeof(fnbuf)); - else if (have_homedir) - snprintf(fnbuf, sizeof(fnbuf), "%s/%s", homedir, ROOT_CRL_FILE); - else - fnbuf[0] = '\0'; - - /* Set the flags to check against the complete CRL chain */ - if (fnbuf[0] != '\0' && - X509_STORE_load_locations(cvstore, fnbuf, NULL) == 1) - { - /* OpenSSL 0.96 does not support X509_V_FLAG_CRL_CHECK */ -#ifdef X509_V_FLAG_CRL_CHECK - X509_STORE_set_flags(cvstore, - X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL); -#else - char *err = SSLerrmessage(ERR_get_error()); - - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("SSL library does not support CRL certificates (file \"%s\")\n"), - fnbuf); - SSLerrfree(err); -#ifdef ENABLE_THREAD_SAFETY - pthread_mutex_unlock(&ssl_config_mutex); -#endif - return -1; -#endif - } - /* if not found, silently ignore; we do not require CRL */ - } -#ifdef ENABLE_THREAD_SAFETY - pthread_mutex_unlock(&ssl_config_mutex); -#endif - + if (have_rootcert) SSL_set_verify(conn->ssl, SSL_VERIFY_PEER, verify_cb); - } - else - { - /* - * stat() failed; assume root file doesn't exist. If sslmode is - * verify-ca or verify-full, this is an error. Otherwise, continue - * without performing any server cert verification. - */ - if (conn->sslmode[0] == 'v') /* "verify-ca" or "verify-full" */ - { - /* - * The only way to reach here with an empty filename is if - * pqGetHomeDirectory failed. That's a sufficiently unusual case - * that it seems worth having a specialized error message for it. - */ - if (fnbuf[0] == '\0') - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("could not get home directory to locate root certificate file\n" - "Either provide the file or change sslmode to disable server certificate verification.\n")); - else - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("root certificate file \"%s\" does not exist\n" - "Either provide the file or change sslmode to disable server certificate verification.\n"), fnbuf); - return -1; - } - } /* * If the OpenSSL version used supports it (from 1.0.0 on) and the user diff --git a/src/test/ssl/Makefile b/src/test/ssl/Makefile index 3d992babff..2b04d82528 100644 --- a/src/test/ssl/Makefile +++ b/src/test/ssl/Makefile @@ -23,7 +23,8 @@ SSLFILES := $(CERTIFICATES:%=ssl/%.key) $(CERTIFICATES:%=ssl/%.crt) \ ssl/client.crl ssl/server.crl ssl/root.crl \ ssl/both-cas-1.crt ssl/both-cas-2.crt \ ssl/root+server_ca.crt ssl/root+server.crl \ - ssl/root+client_ca.crt ssl/root+client.crl + ssl/root+client_ca.crt ssl/root+client.crl \ + ssl/client+client_ca.crt # This target generates all the key and certificate files. sslfiles: $(SSLFILES) @@ -99,6 +100,9 @@ ssl/root+server_ca.crt: ssl/root_ca.crt ssl/server_ca.crt ssl/root+client_ca.crt: ssl/root_ca.crt ssl/client_ca.crt cat $^ > $@ +ssl/client+client_ca.crt: ssl/client.crt ssl/client_ca.crt + cat $^ > $@ + #### CRLs ssl/client.crl: ssl/client-revoked.crt diff --git a/src/test/ssl/README b/src/test/ssl/README index 52bd68f49f..50fa14e287 100644 --- a/src/test/ssl/README +++ b/src/test/ssl/README @@ -65,6 +65,10 @@ root+server_ca root+client_ca Contains root_crt and client_ca.crt. For use as server's "ssl_ca_file". +client+client_ca + Contains client.crt and client_ca.crt in that order. For use as client's + certificate chain. + There are also CRLs for each of the CAs: root.crl, server.crl and client.crl. For convenience, all of these keypairs and certificates are included in the diff --git a/src/test/ssl/ServerSetup.pm b/src/test/ssl/ServerSetup.pm index 4e93184eb0..d312880f8b 100644 --- a/src/test/ssl/ServerSetup.pm +++ b/src/test/ssl/ServerSetup.pm @@ -75,6 +75,7 @@ sub configure_test_server_for_ssl copy_files("ssl/server-*.key", $pgdata); chmod(0600, glob "$pgdata/server-*.key") or die $!; copy_files("ssl/root+client_ca.crt", $pgdata); + copy_files("ssl/root_ca.crt", $pgdata); copy_files("ssl/root+client.crl", $pgdata); # Only accept SSL connections from localhost. Our tests don't depend on this @@ -101,13 +102,14 @@ sub switch_server_cert { my $node = $_[0]; my $certfile = $_[1]; + my $cafile = $_[2] || "root+client_ca"; my $pgdata = $node->data_dir; - diag "Restarting server with certfile \"$certfile\"..."; + diag "Restarting server with certfile \"$certfile\" and cafile \"$cafile\"..."; open SSLCONF, ">$pgdata/sslconfig.conf"; print SSLCONF "ssl=on\n"; - print SSLCONF "ssl_ca_file='root+client_ca.crt'\n"; + print SSLCONF "ssl_ca_file='$cafile.crt'\n"; print SSLCONF "ssl_cert_file='$certfile.crt'\n"; print SSLCONF "ssl_key_file='$certfile.key'\n"; print SSLCONF "ssl_crl_file='root+client.crl'\n"; diff --git a/src/test/ssl/ssl/client+client_ca.crt b/src/test/ssl/ssl/client+client_ca.crt new file mode 100644 index 0000000000..3caada693d --- /dev/null +++ b/src/test/ssl/ssl/client+client_ca.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIBxzCCATACAQEwDQYJKoZIhvcNAQEFBQAwQjFAMD4GA1UEAww3VGVzdCBDQSBm +b3IgUG9zdGdyZVNRTCBTU0wgcmVncmVzc2lvbiB0ZXN0IGNsaWVudCBjZXJ0czAe +Fw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMBYxFDASBgNVBAMMC3NzbHRl +c3R1c2VyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDN3RFl8VWMEBN1Qas0 +w1CFcXdDEbKVNSPsqWHzHIEPoGJv+eUIBK2lQ/Ce8nRCdelO50RsmlbcXBIrjVl6 +BN0RmEeEVclgCdiamYN53LBdc5KWKpKCKn45lCtlZodWt0hNNx1pAmh85jDKpoO9 +ErbCnSU1wODPqnOzdkLU7jBu5QIDAQABMA0GCSqGSIb3DQEBBQUAA4GBABUz+vnu +dD1Q1N/Ezs5DzJeQDtiJb9PNzBHAUPQoXeLvuITcDdyYWc18Yi4fX7gwyD42q2iu +1I0hmm2bNJfujsGbvGYFLuQ4hC2ucAAj2Gm681GhhaNYtfsfHYm9R8GRZFvp40oj +qXpkDkYsPdyVxUyoxJ+M0Ub5VC/k1pQNtIaq +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIICCDCCAXGgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBAMT4wPAYDVQQDDDVUZXN0 +IHJvb3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBzdWl0 +ZTAeFw0xNjA5MTIxNjMwMDFaFw00NDAxMjkxNjMwMDFaMEIxQDA+BgNVBAMMN1Rl +c3QgQ0EgZm9yIFBvc3RncmVTUUwgU1NMIHJlZ3Jlc3Npb24gdGVzdCBjbGllbnQg +Y2VydHMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMI2MXWSb8TZnCLVNYJ+ +19b4noxRmaR1W2zUxl4aTMfiPt9cK06lNY39EPBfjmb7hjxD76w8fLoV/aZ0gOgd +JXFRZvIg7SyM7QVFma0AJAIZayes+ba1odEmBEi378g0mLrjCLqZtBVHfvJxL/6x +6/flSTAn/+09vtELvvLWBePZAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEAlGC24V2TsiSlo9RIboBZTZqd0raUpKkmVbkwKyqcmecoFfCI +TCmoyJLYyUL5/e3dtn/cGDcaqxaO3qxnstxVEMSrlCGfZdZJ2oouXZMpDy9CkeOM +ypCCx9pc4EmP3mvu64f21+dNCXlhM36pZ1IokeS5jk2FIHUda+m5jlk5o6I= +-----END CERTIFICATE----- diff --git a/src/test/ssl/t/001_ssltests.pl b/src/test/ssl/t/001_ssltests.pl index 80e8ea1fe7..dc8e064b25 100644 --- a/src/test/ssl/t/001_ssltests.pl +++ b/src/test/ssl/t/001_ssltests.pl @@ -2,7 +2,7 @@ use warnings; use PostgresNode; use TestLib; -use Test::More tests => 38; +use Test::More tests => 40; use ServerSetup; use File::Copy; @@ -239,3 +239,11 @@ sub test_connect_fails test_connect_fails( "user=ssltestuser sslcert=ssl/client-revoked.crt sslkey=ssl/client-revoked.key" ); + +# intermediate client_ca.crt is provided by client, and isn't in server's ssl_ca_file +switch_server_cert($node, 'server-cn-only', 'root_ca'); +$common_connstr = +"user=ssltestuser dbname=certdb sslkey=ssl/client.key sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR"; + +test_connect_ok("sslmode=require sslcert=ssl/client+client_ca.crt"); +test_connect_fails("sslmode=require sslcert=ssl/client.crt"); From 275bf98601b230e530003ef20193d095b9309c24 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 7 Oct 2016 12:51:52 +0300 Subject: [PATCH 288/871] Clear OpenSSL error queue after failed X509_STORE_load_locations() call. Leaving the error in the error queue used to be harmless, because the X509_STORE_load_locations() call used to be the last step in initialize_SSL(), and we would clear the queue before the next SSL_connect() call. But previous commit moved things around. The symptom was that if a CRL file was not found, and one of the subsequent initialization steps, like loading the client certificate or private key, failed, we would incorrectly print the "no such file" error message from the earlier X509_STORE_load_locations() call as the reason. Backpatch to all supported versions, like the previous patch. --- src/interfaces/libpq/fe-secure-openssl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index c83c1acae4..f474c96f5f 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -996,6 +996,7 @@ initialize_SSL(PGconn *conn) #endif } /* if not found, silently ignore; we do not require CRL */ + ERR_clear_error(); } have_rootcert = true; } From 0d4d7d61850f4f4bc5f6fd0b7a9adb70232aed61 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 7 Oct 2016 14:35:17 +0300 Subject: [PATCH 289/871] Don't allow both --source-server and --source-target args to pg_rewind. They are supposed to be mutually exclusive, but there was no check for that. Michael Banck Discussion: <20161007103414.GD12247@nighthawk.caipicrew.dd-dns.de> --- src/bin/pg_rewind/pg_rewind.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 5fdd4c5605..dd62dd0549 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -162,6 +162,13 @@ main(int argc, char **argv) exit(1); } + if (datadir_source != NULL && connstr_source != NULL) + { + fprintf(stderr, _("%s: only one of --source-pgdata or --source-server can be specified\n"), progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + if (datadir_target == NULL) { fprintf(stderr, _("%s: no target data directory specified (--target-pgdata)\n"), progname); From 4806f26f9e4273888cbf85f42b3bdc5e9d950ba6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 7 Oct 2016 09:51:18 -0400 Subject: [PATCH 290/871] Fix pg_dump to work against pre-9.0 servers again. getBlobs' queries for pre-9.0 servers were broken in two ways: the 7.x/8.x query uses DISTINCT so it can't have unspecified-type NULLs in the target list, and both that query and the 7.0 one failed to provide the correct output column labels, so that the subsequent code to extract data from the PGresult would fail. Back-patch to 9.6 where the breakage was introduced (by commit 23f34fa4b). Amit Langote and Tom Lane Discussion: <0a3e7a0e-37bd-8427-29bd-958135862f0a@lab.ntt.co.jp> --- src/bin/pg_dump/pg_dump.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 299e88788e..fde7f59c3d 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -2875,19 +2875,20 @@ getBlobs(Archive *fout) else if (fout->remoteVersion >= 90000) appendPQExpBuffer(blobQry, "SELECT oid, (%s lomowner) AS rolname, lomacl, " - "NULL AS rlomacl, NULL as initlomacl, " - "NULL as initrlomacl " + "NULL AS rlomacl, NULL AS initlomacl, " + "NULL AS initrlomacl " " FROM pg_largeobject_metadata", username_subquery); else if (fout->remoteVersion >= 70100) appendPQExpBufferStr(blobQry, - "SELECT DISTINCT loid, NULL::oid, NULL, " - "NULL AS rlomacl, NULL AS initlomacl, " - "NULL AS initrlomacl " + "SELECT DISTINCT loid AS oid, " + "NULL::name AS rolname, NULL::oid AS lomacl, " + "NULL::oid AS rlomacl, NULL::oid AS initlomacl, " + "NULL::oid AS initrlomacl " " FROM pg_largeobject"); else appendPQExpBufferStr(blobQry, - "SELECT oid, NULL::oid, NULL, " + "SELECT oid, NULL AS rolname, NULL AS lomacl, " "NULL AS rlomacl, NULL AS initlomacl, " "NULL AS initrlomacl " " FROM pg_class WHERE relkind = 'l'"); @@ -2922,11 +2923,11 @@ getBlobs(Archive *fout) binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl)); binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl)); - if (PQgetisnull(res, i, i_lomacl) && PQgetisnull(res, i, i_rlomacl) && + if (PQgetisnull(res, i, i_lomacl) && + PQgetisnull(res, i, i_rlomacl) && PQgetisnull(res, i, i_initlomacl) && PQgetisnull(res, i, i_initrlomacl)) binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL; - } /* From 17a3a1eb0efc5d84c81e46a26fe6bd21dbe90de9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 7 Oct 2016 11:27:34 -0400 Subject: [PATCH 291/871] Fix python shlib probe for Cygwin. On buildfarm member cockatiel, that library is in /usr/bin. (Possibly we should look at $PATH on this platform?) Per off-list report from Andrew Dunstan. --- config/python.m4 | 10 +++++----- configure | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config/python.m4 b/config/python.m4 index babf5ab1db..b605212bea 100644 --- a/config/python.m4 +++ b/config/python.m4 @@ -130,11 +130,11 @@ else done done fi - # As usual, Windows has its own ideas. c:/Windows/System32 takes the - # place of /usr/lib as a possible default library location, and the - # "lib" prefix might not be there. - if test "$found_shlib" != 1 -a "$PORTNAME" = win32 ; then - for d in "${python_libdir}" "${python_configdir}" c:/Windows/System32 + # As usual, Windows has its own ideas. Possible default library + # locations include c:/Windows/System32 and (for Cygwin) /usr/bin, + # and the "lib" prefix might not be there. + if test "$found_shlib" != 1 -a \( "$PORTNAME" = win32 -o "$PORTNAME" = cygwin \); then + for d in "${python_libdir}" "${python_configdir}" c:/Windows/System32 /usr/bin do for f in "$d/lib${ldlibrary}.dll" "$d/${ldlibrary}.dll" ; do if test -e "$f"; then diff --git a/configure b/configure index 204daa056d..ceaa3ba5f8 100755 --- a/configure +++ b/configure @@ -7673,11 +7673,11 @@ else done done fi - # As usual, Windows has its own ideas. c:/Windows/System32 takes the - # place of /usr/lib as a possible default library location, and the - # "lib" prefix might not be there. - if test "$found_shlib" != 1 -a "$PORTNAME" = win32 ; then - for d in "${python_libdir}" "${python_configdir}" c:/Windows/System32 + # As usual, Windows has its own ideas. Possible default library + # locations include c:/Windows/System32 and (for Cygwin) /usr/bin, + # and the "lib" prefix might not be there. + if test "$found_shlib" != 1 -a \( "$PORTNAME" = win32 -o "$PORTNAME" = cygwin \); then + for d in "${python_libdir}" "${python_configdir}" c:/Windows/System32 /usr/bin do for f in "$d/lib${ldlibrary}.dll" "$d/${ldlibrary}.dll" ; do if test -e "$f"; then From d668b03378c28b927d0ba458681ca1b4c1e18e53 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 7 Oct 2016 21:48:21 +0300 Subject: [PATCH 292/871] Make TAP test suites to work, when @INC does not contain current dir. Recent Perl and/or new Linux distributions are starting to remove "." from the @INC list by default. That breaks pg_rewind and ssl test suites, which use helper perl modules that reside in the same directory. To fix, add the current source directory explicitly to prove's include dir. The vcregress.pl script probably also needs something like this, but I wasn't able to remove '.' from @INC on Windows to test this, and don't want to try doing that blindly. Discussion: <20160908204529.flg6nivjuwp5vaoy@alap3.anarazel.de> --- src/Makefile.global.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Makefile.global.in b/src/Makefile.global.in index c211a2d2e7..e1e2c0adaf 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -332,7 +332,9 @@ endif endif PROVE = @PROVE@ -PG_PROVE_FLAGS = -I $(top_srcdir)/src/test/perl/ +# There are common routines in src/test/perl, and some test suites have +# extra perl modules in their own directory. +PG_PROVE_FLAGS = -I $(top_srcdir)/src/test/perl/ -I $(srcdir) PROVE_FLAGS = --verbose # prepend to path if already set, else just set it From 0aec7f9aec8b828e074b8f2f3cbea2ec3e5c0209 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 7 Oct 2016 23:56:42 +0300 Subject: [PATCH 293/871] Remove bogus mapping from UTF-8 to SJIS conversion table. 0xc19c is not a valid UTF-8 byte sequence. It doesn't do any harm, AFAICS, but it's surely not intentional. No backpatching though, just to be sure. In the passing, also add a file header comment to the file, like the UCS_to_SJIS.pl script would produce. (The file was originally created with UCS_to_SJIS.pl, but has been modified by hand since then. That's questionable, but I'll leave fixing that for later.) Kyotaro Horiguchi Discussion: <20160907.155050.233844095.horiguchi.kyotaro@lab.ntt.co.jp> --- src/backend/utils/mb/Unicode/utf8_to_sjis.map | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/mb/Unicode/utf8_to_sjis.map b/src/backend/utils/mb/Unicode/utf8_to_sjis.map index bcb76c9150..fb0566a1db 100644 --- a/src/backend/utils/mb/Unicode/utf8_to_sjis.map +++ b/src/backend/utils/mb/Unicode/utf8_to_sjis.map @@ -1,5 +1,6 @@ -static const pg_utf_to_local ULmapSJIS[ 7398 ] = { - {0xc19c, 0x815f}, +/* src/backend/utils/mb/Unicode/utf8_to_sjis.map */ + +static const pg_utf_to_local ULmapSJIS[ 7397 ] = { {0xc2a2, 0x8191}, {0xc2a3, 0x8192}, {0xc2a5, 0x5c}, From b0779abb3add11d4dd745779dd81ea8ecdd00a1d Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Fri, 7 Oct 2016 16:55:15 -0700 Subject: [PATCH 294/871] Fix fallback implementation of pg_atomic_write_u32(). I somehow had assumed that in the spinlock (in turn possibly using semaphores) based fallback atomics implementation 32 bit writes could be done without a lock. As far as the write goes that's correct, since postgres supports only platforms with single-copy atomicity for aligned 32bit writes. But writing without holding the spinlock breaks read-modify-write operations like pg_atomic_compare_exchange_u32(), since they'll potentially "miss" a concurrent write, which can't happen in actual hardware implementations. In 9.6+ when using the fallback atomics implementation this could lead to buffer header locks not being properly marked as released, and potentially some related state corruption. I don't see a related danger in 9.5 (earliest release with the API), because pg_atomic_write_u32() wasn't used in a concurrent manner there. The state variable of local buffers, before this change, were manipulated using pg_atomic_write_u32(), to avoid unnecessary synchronization overhead. As that'd not be the case anymore, introduce and use pg_atomic_unlocked_write_u32(), which does not correctly interact with RMW operations. This bug only caused issues when postgres is compiled on platforms without atomics support (i.e. no common new platform), or when compiled with --disable-atomics, which explains why this wasn't noticed in testing. Reported-By: Tom Lane Discussion: <14947.1475690465@sss.pgh.pa.us> Backpatch: 9.5-, where the atomic operations API was introduced. --- src/backend/port/atomics.c | 13 +++++++++++++ src/backend/storage/buffer/bufmgr.c | 6 +++--- src/backend/storage/buffer/localbuf.c | 16 ++++++++-------- src/include/port/atomics.h | 25 +++++++++++++++++++++++-- src/include/port/atomics/fallback.h | 3 +++ src/include/port/atomics/generic.h | 9 +++++++++ src/include/storage/buf_internals.h | 3 ++- 7 files changed, 61 insertions(+), 14 deletions(-) diff --git a/src/backend/port/atomics.c b/src/backend/port/atomics.c index 42169a33cf..d5970e42b0 100644 --- a/src/backend/port/atomics.c +++ b/src/backend/port/atomics.c @@ -104,6 +104,19 @@ pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_) ptr->value = val_; } +void +pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + /* + * One might think that an unlocked write doesn't need to acquire the + * spinlock, but one would be wrong. Even an unlocked write has to cause a + * concurrent pg_atomic_compare_exchange_u32() (et al) to fail. + */ + SpinLockAcquire((slock_t *) &ptr->sema); + ptr->value = val; + SpinLockRelease((slock_t *) &ptr->sema); +} + bool pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 *expected, uint32 newval) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 2b63cd3ef1..df4c9d7109 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -821,7 +821,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Assert(buf_state & BM_VALID); buf_state &= ~BM_VALID; - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); } else { @@ -941,7 +941,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, uint32 buf_state = pg_atomic_read_u32(&bufHdr->state); buf_state |= BM_VALID; - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); } else { @@ -3167,7 +3167,7 @@ FlushRelationBuffers(Relation rel) false); buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED); - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); /* Pop the error context stack */ error_context_stack = errcallback.previous; diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c index ca2388789d..fa961ad9c8 100644 --- a/src/backend/storage/buffer/localbuf.c +++ b/src/backend/storage/buffer/localbuf.c @@ -138,7 +138,7 @@ LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT) { buf_state += BUF_USAGECOUNT_ONE; - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); } } LocalRefCount[b]++; @@ -181,7 +181,7 @@ LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, if (BUF_STATE_GET_USAGECOUNT(buf_state) > 0) { buf_state -= BUF_USAGECOUNT_ONE; - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); trycounter = NLocBuffer; } else @@ -222,7 +222,7 @@ LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, /* Mark not-dirty now in case we error out below */ buf_state &= ~BM_DIRTY; - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); pgBufferUsage.local_blks_written++; } @@ -249,7 +249,7 @@ LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, /* mark buffer invalid just in case hash insert fails */ CLEAR_BUFFERTAG(bufHdr->tag); buf_state &= ~(BM_VALID | BM_TAG_VALID); - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); } hresult = (LocalBufferLookupEnt *) @@ -266,7 +266,7 @@ LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, buf_state |= BM_TAG_VALID; buf_state &= ~BUF_USAGECOUNT_MASK; buf_state += BUF_USAGECOUNT_ONE; - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); *foundPtr = FALSE; return bufHdr; @@ -302,7 +302,7 @@ MarkLocalBufferDirty(Buffer buffer) buf_state |= BM_DIRTY; - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); } /* @@ -351,7 +351,7 @@ DropRelFileNodeLocalBuffers(RelFileNode rnode, ForkNumber forkNum, CLEAR_BUFFERTAG(bufHdr->tag); buf_state &= ~BUF_FLAG_MASK; buf_state &= ~BUF_USAGECOUNT_MASK; - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); } } } @@ -395,7 +395,7 @@ DropRelFileNodeAllLocalBuffers(RelFileNode rnode) CLEAR_BUFFERTAG(bufHdr->tag); buf_state &= ~BUF_FLAG_MASK; buf_state &= ~BUF_USAGECOUNT_MASK; - pg_atomic_write_u32(&bufHdr->state, buf_state); + pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); } } } diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index f7884d72c6..4e15ff8764 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -255,10 +255,12 @@ pg_atomic_read_u32(volatile pg_atomic_uint32 *ptr) } /* - * pg_atomic_write_u32 - unlocked write to atomic variable. + * pg_atomic_write_u32 - write to atomic variable. * * The write is guaranteed to succeed as a whole, i.e. it's not possible to - * observe a partial write for any reader. + * observe a partial write for any reader. Note that this correctly interacts + * with pg_atomic_compare_exchange_u32, in contrast to + * pg_atomic_unlocked_write_u32(). * * No barrier semantics. */ @@ -270,6 +272,25 @@ pg_atomic_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val) pg_atomic_write_u32_impl(ptr, val); } +/* + * pg_atomic_unlocked_write_u32 - unlocked write to atomic variable. + * + * The write is guaranteed to succeed as a whole, i.e. it's not possible to + * observe a partial write for any reader. But note that writing this way is + * not guaranteed to correctly interact with read-modify-write operations like + * pg_atomic_compare_exchange_u32. This should only be used in cases where + * minor performance regressions due to atomics emulation are unacceptable. + * + * No barrier semantics. + */ +static inline void +pg_atomic_unlocked_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + AssertPointerAlignment(ptr, 4); + + pg_atomic_unlocked_write_u32_impl(ptr, val); +} + /* * pg_atomic_exchange_u32 - exchange newval with current value * diff --git a/src/include/port/atomics/fallback.h b/src/include/port/atomics/fallback.h index bdaa795abe..2290fff196 100644 --- a/src/include/port/atomics/fallback.h +++ b/src/include/port/atomics/fallback.h @@ -133,6 +133,9 @@ pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) #define PG_HAVE_ATOMIC_INIT_U32 extern void pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_); +#define PG_HAVE_ATOMIC_WRITE_U32 +extern void pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val); + #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 extern bool pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 *expected, uint32 newval); diff --git a/src/include/port/atomics/generic.h b/src/include/port/atomics/generic.h index 32a01136e6..1acd19242b 100644 --- a/src/include/port/atomics/generic.h +++ b/src/include/port/atomics/generic.h @@ -58,6 +58,15 @@ pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) } #endif +#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 +#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 +static inline void +pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + ptr->value = val; +} +#endif + /* * provide fallback for test_and_set using atomic_exchange if available */ diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index e0dfb2f5b0..c7da9f6ac2 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -168,7 +168,8 @@ typedef struct buftag * We use this same struct for local buffer headers, but the locks are not * used and not all of the flag bits are useful either. To avoid unnecessary * overhead, manipulations of the state field should be done without actual - * atomic operations (i.e. only pg_atomic_read/write). + * atomic operations (i.e. only pg_atomic_read_u32() and + * pg_atomic_unlocked_write_u32()). * * Be careful to avoid increasing the size of the struct when adding or * reordering members. Keeping it below 64 bytes (the most common CPU From 8811f5d3a4a1ebf79ccb00da336d70041b003dd2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 7 Oct 2016 21:12:25 -0400 Subject: [PATCH 295/871] libpqwalreceiver needs to link with libintl when using --enable-nls. The need for this was previously obscured even on picky platforms by the hack we used to support direct cross-module references in the transforms contrib modules. Now that that hack is gone, the undefined symbol is exposed, as reported by Robert Haas. Back-patch to 9.5 where we started to use -Wl,-undefined,dynamic_lookup. I'm a bit surprised that the older branches don't seem to contain any gettext references in this module, but since they don't fail at build time, they must not. (We might be able to get away with leaving this alone in 9.5/9.6, but I think it's cleaner if the reference gets resolved at link time.) Report: --- src/backend/replication/libpqwalreceiver/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/replication/libpqwalreceiver/Makefile b/src/backend/replication/libpqwalreceiver/Makefile index 6c95c1ce97..a7a5fe1ed2 100644 --- a/src/backend/replication/libpqwalreceiver/Makefile +++ b/src/backend/replication/libpqwalreceiver/Makefile @@ -15,7 +15,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS) OBJS = libpqwalreceiver.o $(WIN32RES) -SHLIB_LINK = $(libpq) +SHLIB_LINK = $(libpq) $(filter -lintl, $(LIBS)) SHLIB_PREREQS = submake-libpq PGFILEDESC = "libpqwalreceiver - receive WAL during streaming replication" NAME = libpqwalreceiver From e55a946a81c6648c5ff3a0ccdda242f639e33c6f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 8 Oct 2016 19:29:27 -0400 Subject: [PATCH 296/871] Fix two bugs in merging of inherited CHECK constraints. Historically, we've allowed users to add a CHECK constraint to a child table and then add an identical CHECK constraint to the parent. This results in "merging" the two constraints so that the pre-existing child constraint ends up with both conislocal = true and coninhcount > 0. However, if you tried to do it in the other order, you got a duplicate constraint error. This is problematic for pg_dump, which needs to issue separated ADD CONSTRAINT commands in some cases, but has no good way to ensure that the constraints will be added in the required order. And it's more than a bit arbitrary, too. The goal of complaining about duplicated ADD CONSTRAINT commands can be served if we reject the case of adding a constraint when the existing one already has conislocal = true; but if it has conislocal = false, let's just make the ADD CONSTRAINT set conislocal = true. In this way, either order of adding the constraints has the same end result. Another problem was that the code allowed creation of a parent constraint marked convalidated that is merged with a child constraint that is !convalidated. In this case, an inheritance scan of the parent table could emit some rows violating the constraint condition, which would be an unexpected result given the marking of the parent constraint as validated. Hence, forbid merging of constraints in this case. (Note: valid child and not-valid parent seems fine, so continue to allow that.) Per report from Benedikt Grundmann. Back-patch to 9.2 where we introduced possibly-not-valid check constraints. The second bug obviously doesn't apply before that, and I think the first doesn't either, because pg_dump only gets into this situation when dealing with not-valid constraints. Report: Discussion: <22108.1475874586@sss.pgh.pa.us> --- src/backend/catalog/heap.c | 40 +++++++++++++---- src/backend/commands/tablecmds.c | 13 +++++- src/test/regress/expected/inherit.out | 50 ++++++++++++++++++++++ src/test/regress/expected/sanity_check.out | 2 + src/test/regress/sql/inherit.sql | 39 +++++++++++++++++ 5 files changed, 135 insertions(+), 9 deletions(-) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index dbd609493f..ea06a5739a 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -105,6 +105,7 @@ static void StoreConstraints(Relation rel, List *cooked_constraints, bool is_internal); static bool MergeWithExistingConstraint(Relation rel, char *ccname, Node *expr, bool allow_merge, bool is_local, + bool is_initially_valid, bool is_no_inherit); static void SetRelationNumChecks(Relation rel, int numchecks); static Node *cookConstraint(ParseState *pstate, @@ -2301,6 +2302,7 @@ AddRelationNewConstraints(Relation rel, */ if (MergeWithExistingConstraint(rel, ccname, expr, allow_merge, is_local, + cdef->initially_valid, cdef->is_no_inherit)) continue; } @@ -2389,6 +2391,7 @@ AddRelationNewConstraints(Relation rel, static bool MergeWithExistingConstraint(Relation rel, char *ccname, Node *expr, bool allow_merge, bool is_local, + bool is_initially_valid, bool is_no_inherit) { bool found; @@ -2436,22 +2439,47 @@ MergeWithExistingConstraint(Relation rel, char *ccname, Node *expr, if (equal(expr, stringToNode(TextDatumGetCString(val)))) found = true; } + + /* + * If the existing constraint is purely inherited (no local + * definition) then interpret addition of a local constraint as a + * legal merge. This allows ALTER ADD CONSTRAINT on parent and + * child tables to be given in either order with same end state. + */ + if (is_local && !con->conislocal) + allow_merge = true; + if (!found || !allow_merge) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("constraint \"%s\" for relation \"%s\" already exists", ccname, RelationGetRelationName(rel)))); - tup = heap_copytuple(tup); - con = (Form_pg_constraint) GETSTRUCT(tup); - - /* If the constraint is "no inherit" then cannot merge */ + /* If the child constraint is "no inherit" then cannot merge */ if (con->connoinherit) ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), errmsg("constraint \"%s\" conflicts with non-inherited constraint on relation \"%s\"", ccname, RelationGetRelationName(rel)))); + /* + * If the child constraint is "not valid" then cannot merge with a + * valid parent constraint + */ + if (is_initially_valid && !con->convalidated) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("constraint \"%s\" conflicts with NOT VALID constraint on relation \"%s\"", + ccname, RelationGetRelationName(rel)))); + + /* OK to update the tuple */ + ereport(NOTICE, + (errmsg("merging constraint \"%s\" with inherited definition", + ccname))); + + tup = heap_copytuple(tup); + con = (Form_pg_constraint) GETSTRUCT(tup); + if (is_local) con->conislocal = true; else @@ -2461,10 +2489,6 @@ MergeWithExistingConstraint(Relation rel, char *ccname, Node *expr, Assert(is_local); con->connoinherit = true; } - /* OK to update the tuple */ - ereport(NOTICE, - (errmsg("merging constraint \"%s\" with inherited definition", - ccname))); simple_heap_update(conDesc, &tup->t_self, tup); CatalogUpdateIndexes(conDesc, tup); break; diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index d31276284c..f822ed9597 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -10373,7 +10373,7 @@ MergeConstraintsIntoExisting(Relation child_rel, Relation parent_rel) RelationGetRelationName(child_rel), NameStr(parent_con->conname)))); - /* If the constraint is "no inherit" then cannot merge */ + /* If the child constraint is "no inherit" then cannot merge */ if (child_con->connoinherit) ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), @@ -10381,6 +10381,17 @@ MergeConstraintsIntoExisting(Relation child_rel, Relation parent_rel) NameStr(child_con->conname), RelationGetRelationName(child_rel)))); + /* + * If the child constraint is "not valid" then cannot merge with a + * valid parent constraint + */ + if (parent_con->convalidated && !child_con->convalidated) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("constraint \"%s\" conflicts with NOT VALID constraint on child table \"%s\"", + NameStr(child_con->conname), + RelationGetRelationName(child_rel)))); + /* * OK, bump the child constraint's inheritance count. (If we fail * later on, this change will just roll back.) diff --git a/src/test/regress/expected/inherit.out b/src/test/regress/expected/inherit.out index d8b5b1d44e..9d374fe6c4 100644 --- a/src/test/regress/expected/inherit.out +++ b/src/test/regress/expected/inherit.out @@ -1090,6 +1090,56 @@ Inherits: test_foreign_constraints DROP TABLE test_foreign_constraints_inh; DROP TABLE test_foreign_constraints; DROP TABLE test_primary_constraints; +-- Test that parent and child CHECK constraints can be created in either order +create table p1(f1 int); +create table p1_c1() inherits(p1); +alter table p1 add constraint inh_check_constraint1 check (f1 > 0); +alter table p1_c1 add constraint inh_check_constraint1 check (f1 > 0); +NOTICE: merging constraint "inh_check_constraint1" with inherited definition +alter table p1_c1 add constraint inh_check_constraint2 check (f1 < 10); +alter table p1 add constraint inh_check_constraint2 check (f1 < 10); +NOTICE: merging constraint "inh_check_constraint2" with inherited definition +select conrelid::regclass::text as relname, conname, conislocal, coninhcount +from pg_constraint where conname like 'inh\_check\_constraint%' +order by 1, 2; + relname | conname | conislocal | coninhcount +---------+-----------------------+------------+------------- + p1 | inh_check_constraint1 | t | 0 + p1 | inh_check_constraint2 | t | 0 + p1_c1 | inh_check_constraint1 | t | 1 + p1_c1 | inh_check_constraint2 | t | 1 +(4 rows) + +drop table p1 cascade; +NOTICE: drop cascades to table p1_c1 +-- Test that a valid child can have not-valid parent, but not vice versa +create table invalid_check_con(f1 int); +create table invalid_check_con_child() inherits(invalid_check_con); +alter table invalid_check_con_child add constraint inh_check_constraint check(f1 > 0) not valid; +alter table invalid_check_con add constraint inh_check_constraint check(f1 > 0); -- fail +ERROR: constraint "inh_check_constraint" conflicts with NOT VALID constraint on relation "invalid_check_con_child" +alter table invalid_check_con_child drop constraint inh_check_constraint; +insert into invalid_check_con values(0); +alter table invalid_check_con_child add constraint inh_check_constraint check(f1 > 0); +alter table invalid_check_con add constraint inh_check_constraint check(f1 > 0) not valid; +NOTICE: merging constraint "inh_check_constraint" with inherited definition +insert into invalid_check_con values(0); -- fail +ERROR: new row for relation "invalid_check_con" violates check constraint "inh_check_constraint" +DETAIL: Failing row contains (0). +insert into invalid_check_con_child values(0); -- fail +ERROR: new row for relation "invalid_check_con_child" violates check constraint "inh_check_constraint" +DETAIL: Failing row contains (0). +select conrelid::regclass::text as relname, conname, + convalidated, conislocal, coninhcount, connoinherit +from pg_constraint where conname like 'inh\_check\_constraint%' +order by 1, 2; + relname | conname | convalidated | conislocal | coninhcount | connoinherit +-------------------------+----------------------+--------------+------------+-------------+-------------- + invalid_check_con | inh_check_constraint | f | t | 0 | f + invalid_check_con_child | inh_check_constraint | t | t | 1 | f +(2 rows) + +-- We don't drop the invalid_check_con* tables, to test dump/reload with -- -- Test parameterized append plans for inheritance trees -- diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index 1c087a3903..b1ebcf60d2 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -62,6 +62,8 @@ int2_tbl|f int4_tbl|f int8_tbl|f interval_tbl|f +invalid_check_con|f +invalid_check_con_child|f iportaltest|f kd_point_tbl|t line_tbl|f diff --git a/src/test/regress/sql/inherit.sql b/src/test/regress/sql/inherit.sql index b307a5049b..6b1df754a6 100644 --- a/src/test/regress/sql/inherit.sql +++ b/src/test/regress/sql/inherit.sql @@ -334,6 +334,45 @@ DROP TABLE test_foreign_constraints_inh; DROP TABLE test_foreign_constraints; DROP TABLE test_primary_constraints; +-- Test that parent and child CHECK constraints can be created in either order +create table p1(f1 int); +create table p1_c1() inherits(p1); + +alter table p1 add constraint inh_check_constraint1 check (f1 > 0); +alter table p1_c1 add constraint inh_check_constraint1 check (f1 > 0); + +alter table p1_c1 add constraint inh_check_constraint2 check (f1 < 10); +alter table p1 add constraint inh_check_constraint2 check (f1 < 10); + +select conrelid::regclass::text as relname, conname, conislocal, coninhcount +from pg_constraint where conname like 'inh\_check\_constraint%' +order by 1, 2; + +drop table p1 cascade; + +-- Test that a valid child can have not-valid parent, but not vice versa +create table invalid_check_con(f1 int); +create table invalid_check_con_child() inherits(invalid_check_con); + +alter table invalid_check_con_child add constraint inh_check_constraint check(f1 > 0) not valid; +alter table invalid_check_con add constraint inh_check_constraint check(f1 > 0); -- fail +alter table invalid_check_con_child drop constraint inh_check_constraint; + +insert into invalid_check_con values(0); + +alter table invalid_check_con_child add constraint inh_check_constraint check(f1 > 0); +alter table invalid_check_con add constraint inh_check_constraint check(f1 > 0) not valid; + +insert into invalid_check_con values(0); -- fail +insert into invalid_check_con_child values(0); -- fail + +select conrelid::regclass::text as relname, conname, + convalidated, conislocal, coninhcount, connoinherit +from pg_constraint where conname like 'inh\_check\_constraint%' +order by 1, 2; + +-- We don't drop the invalid_check_con* tables, to test dump/reload with + -- -- Test parameterized append plans for inheritance trees -- From ac4a9d92fcb6869e757cc729dca2ca5ccf94b185 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 9 Oct 2016 12:49:37 -0400 Subject: [PATCH 297/871] Fix incorrect handling of polymorphic aggregates used as window functions. The transfunction was told that its first argument and result were of the window function output type, not the aggregate state type. This'd only matter if the transfunction consults get_fn_expr_argtype, which typically only polymorphic functions would do. Although we have several regression tests around polymorphic aggs, none of them detected this mistake --- in fact, they still didn't fail when I injected the same mistake into nodeAgg.c. So add some more tests covering both plain agg and window-function-agg cases. Per report from Sebastian Luque. Back-patch to 9.6 where the error was introduced (by sloppy refactoring in commit 804163bc2, looks like). Report: <87int2qkat.fsf@gmail.com> --- src/backend/executor/nodeWindowAgg.c | 2 +- src/test/regress/expected/polymorphism.out | 55 ++++++++++++++++++++++ src/test/regress/sql/polymorphism.sql | 22 +++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c index 371548ceb3..96c8527eb3 100644 --- a/src/backend/executor/nodeWindowAgg.c +++ b/src/backend/executor/nodeWindowAgg.c @@ -2218,7 +2218,7 @@ initialize_peragg(WindowAggState *winstate, WindowFunc *wfunc, numArguments, 0, /* no ordered-set window functions yet */ false, /* no variadic window functions yet */ - wfunc->wintype, + aggtranstype, wfunc->inputcollid, transfn_oid, invtransfn_oid, diff --git a/src/test/regress/expected/polymorphism.out b/src/test/regress/expected/polymorphism.out index ddf45cf597..68b88d33a1 100644 --- a/src/test/regress/expected/polymorphism.out +++ b/src/test/regress/expected/polymorphism.out @@ -635,6 +635,61 @@ create aggregate build_group(int8, integer) ( SFUNC = add_group, STYPE = int8[] ); +-- check proper resolution of data types for polymorphic transfn/finalfn +create function first_el(anyarray) returns anyelement as +'select $1[1]' language sql strict immutable; +create aggregate first_el_agg_f8(float8) ( + SFUNC = array_append, + STYPE = float8[], + FINALFUNC = first_el +); +create aggregate first_el_agg_any(anyelement) ( + SFUNC = array_append, + STYPE = anyarray, + FINALFUNC = first_el +); +select first_el_agg_f8(x::float8) from generate_series(1,10) x; + first_el_agg_f8 +----------------- + 1 +(1 row) + +select first_el_agg_any(x) from generate_series(1,10) x; + first_el_agg_any +------------------ + 1 +(1 row) + +select first_el_agg_f8(x::float8) over(order by x) from generate_series(1,10) x; + first_el_agg_f8 +----------------- + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 +(10 rows) + +select first_el_agg_any(x) over(order by x) from generate_series(1,10) x; + first_el_agg_any +------------------ + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 +(10 rows) + -- check that we can apply functions taking ANYARRAY to pg_stats select distinct array_ndims(histogram_bounds) from pg_stats where histogram_bounds is not null; diff --git a/src/test/regress/sql/polymorphism.sql b/src/test/regress/sql/polymorphism.sql index 72f6cb5e7c..45ae7a23aa 100644 --- a/src/test/regress/sql/polymorphism.sql +++ b/src/test/regress/sql/polymorphism.sql @@ -443,6 +443,28 @@ create aggregate build_group(int8, integer) ( STYPE = int8[] ); +-- check proper resolution of data types for polymorphic transfn/finalfn + +create function first_el(anyarray) returns anyelement as +'select $1[1]' language sql strict immutable; + +create aggregate first_el_agg_f8(float8) ( + SFUNC = array_append, + STYPE = float8[], + FINALFUNC = first_el +); + +create aggregate first_el_agg_any(anyelement) ( + SFUNC = array_append, + STYPE = anyarray, + FINALFUNC = first_el +); + +select first_el_agg_f8(x::float8) from generate_series(1,10) x; +select first_el_agg_any(x) from generate_series(1,10) x; +select first_el_agg_f8(x::float8) over(order by x) from generate_series(1,10) x; +select first_el_agg_any(x) over(order by x) from generate_series(1,10) x; + -- check that we can apply functions taking ANYARRAY to pg_stats select distinct array_ndims(histogram_bounds) from pg_stats where histogram_bounds is not null; From ecb0d20a9d2e09b7112d3b192047f711f9ff7e59 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 9 Oct 2016 18:03:45 -0400 Subject: [PATCH 298/871] Use unnamed POSIX semaphores, if available, on Linux and FreeBSD. We've had support for using unnamed POSIX semaphores instead of System V semaphores for quite some time, but it was not used by default on any platform. Since many systems have rather small limits on the number of SysV semaphores allowed, it seems desirable to switch to POSIX semaphores where they're available and don't create performance or kernel resource problems. Experimentation by me shows that unnamed POSIX semaphores are at least as good as SysV semaphores on Linux, and we previously had a report from Maksym Sobolyev that FreeBSD is significantly worse with SysV semaphores than POSIX ones. So adjust those two platforms to use unnamed POSIX semaphores, if configure can find the necessary library functions. If this goes well, we may switch other platforms as well, but it would be advisable to test them individually first. It's not currently contemplated that we'd encourage users to select a semaphore API for themselves, but anyone who wants to experiment can add PREFERRED_SEMAPHORES=UNNAMED_POSIX (or NAMED_POSIX, or SYSV) to their configure command line to do so. I also tweaked configure to report which API it's selected, mainly so that we can tell that from buildfarm reports. I did not touch the user documentation's discussion about semaphores; that will need some adjustment once the dust settles. Discussion: <8536.1475704230@sss.pgh.pa.us> --- configure | 125 +++++++++++++++++++++++++++++++++++++++++++ configure.in | 13 +++++ src/template/freebsd | 5 ++ src/template/linux | 5 ++ 4 files changed, 148 insertions(+) diff --git a/configure b/configure index ceaa3ba5f8..1d94256a9e 100755 --- a/configure +++ b/configure @@ -14855,23 +14855,148 @@ fi # Select semaphore implementation type. if test "$PORTNAME" != "win32"; then + if test x"$PREFERRED_SEMAPHORES" = x"NAMED_POSIX" ; then + # Need sem_open for this + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing sem_open" >&5 +$as_echo_n "checking for library containing sem_open... " >&6; } +if ${ac_cv_search_sem_open+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sem_open (); +int +main () +{ +return sem_open (); + ; + return 0; +} +_ACEOF +for ac_lib in '' rt pthread; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_sem_open=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_sem_open+:} false; then : + break +fi +done +if ${ac_cv_search_sem_open+:} false; then : + +else + ac_cv_search_sem_open=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_sem_open" >&5 +$as_echo "$ac_cv_search_sem_open" >&6; } +ac_res=$ac_cv_search_sem_open +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + USE_NAMED_POSIX_SEMAPHORES=1 +fi + + fi + if test x"$PREFERRED_SEMAPHORES" = x"UNNAMED_POSIX" ; then + # Need sem_init for this + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing sem_init" >&5 +$as_echo_n "checking for library containing sem_init... " >&6; } +if ${ac_cv_search_sem_init+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char sem_init (); +int +main () +{ +return sem_init (); + ; + return 0; +} +_ACEOF +for ac_lib in '' rt pthread; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_sem_init=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_sem_init+:} false; then : + break +fi +done +if ${ac_cv_search_sem_init+:} false; then : + +else + ac_cv_search_sem_init=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_sem_init" >&5 +$as_echo "$ac_cv_search_sem_init" >&6; } +ac_res=$ac_cv_search_sem_init +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + USE_UNNAMED_POSIX_SEMAPHORES=1 +fi + + fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking which semaphore API to use" >&5 +$as_echo_n "checking which semaphore API to use... " >&6; } if test x"$USE_NAMED_POSIX_SEMAPHORES" = x"1" ; then $as_echo "#define USE_NAMED_POSIX_SEMAPHORES 1" >>confdefs.h SEMA_IMPLEMENTATION="src/backend/port/posix_sema.c" + sematype="named POSIX" else if test x"$USE_UNNAMED_POSIX_SEMAPHORES" = x"1" ; then $as_echo "#define USE_UNNAMED_POSIX_SEMAPHORES 1" >>confdefs.h SEMA_IMPLEMENTATION="src/backend/port/posix_sema.c" + sematype="unnamed POSIX" else $as_echo "#define USE_SYSV_SEMAPHORES 1" >>confdefs.h SEMA_IMPLEMENTATION="src/backend/port/sysv_sema.c" + sematype="System V" fi + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $sematype" >&5 +$as_echo "$sematype" >&6; } fi else diff --git a/configure.in b/configure.in index 40f3c093f1..9ace625029 100644 --- a/configure.in +++ b/configure.in @@ -1939,17 +1939,30 @@ AC_SUBST(PG_CRC32C_OBJS) # Select semaphore implementation type. if test "$PORTNAME" != "win32"; then + if test x"$PREFERRED_SEMAPHORES" = x"NAMED_POSIX" ; then + # Need sem_open for this + AC_SEARCH_LIBS(sem_open, [rt pthread], [USE_NAMED_POSIX_SEMAPHORES=1]) + fi + if test x"$PREFERRED_SEMAPHORES" = x"UNNAMED_POSIX" ; then + # Need sem_init for this + AC_SEARCH_LIBS(sem_init, [rt pthread], [USE_UNNAMED_POSIX_SEMAPHORES=1]) + fi + AC_MSG_CHECKING([which semaphore API to use]) if test x"$USE_NAMED_POSIX_SEMAPHORES" = x"1" ; then AC_DEFINE(USE_NAMED_POSIX_SEMAPHORES, 1, [Define to select named POSIX semaphores.]) SEMA_IMPLEMENTATION="src/backend/port/posix_sema.c" + sematype="named POSIX" else if test x"$USE_UNNAMED_POSIX_SEMAPHORES" = x"1" ; then AC_DEFINE(USE_UNNAMED_POSIX_SEMAPHORES, 1, [Define to select unnamed POSIX semaphores.]) SEMA_IMPLEMENTATION="src/backend/port/posix_sema.c" + sematype="unnamed POSIX" else AC_DEFINE(USE_SYSV_SEMAPHORES, 1, [Define to select SysV-style semaphores.]) SEMA_IMPLEMENTATION="src/backend/port/sysv_sema.c" + sematype="System V" fi + AC_MSG_RESULT([$sematype]) fi else AC_DEFINE(USE_WIN32_SEMAPHORES, 1, [Define to select Win32-style semaphores.]) diff --git a/src/template/freebsd b/src/template/freebsd index 772e2f1a5f..a82d5a494c 100644 --- a/src/template/freebsd +++ b/src/template/freebsd @@ -1 +1,6 @@ # src/template/freebsd + +# Prefer unnamed POSIX semaphores if available, unless user overrides choice +if test x"$PREFERRED_SEMAPHORES" = x"" ; then + PREFERRED_SEMAPHORES=UNNAMED_POSIX +fi diff --git a/src/template/linux b/src/template/linux index 3eb5ad2428..f820bf7280 100644 --- a/src/template/linux +++ b/src/template/linux @@ -1,5 +1,10 @@ # src/template/linux +# Prefer unnamed POSIX semaphores if available, unless user overrides choice +if test x"$PREFERRED_SEMAPHORES" = x"" ; then + PREFERRED_SEMAPHORES=UNNAMED_POSIX +fi + # Force _GNU_SOURCE on; plperl is broken with Perl 5.8.0 otherwise CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" From 52f0142eb4c755d1f4c390464769d35f21e998c3 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sun, 9 Oct 2016 12:00:00 -0400 Subject: [PATCH 299/871] Add a noreturn attribute to help static analyzers --- src/backend/utils/fmgr/dfmgr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/fmgr/dfmgr.c b/src/backend/utils/fmgr/dfmgr.c index 6f70813a6d..905b981b6a 100644 --- a/src/backend/utils/fmgr/dfmgr.c +++ b/src/backend/utils/fmgr/dfmgr.c @@ -65,7 +65,7 @@ char *Dynamic_library_path; static void *internal_load_library(const char *libname); static void incompatible_module_error(const char *libname, - const Pg_magic_struct *module_magic_data); + const Pg_magic_struct *module_magic_data) pg_attribute_noreturn(); static void internal_unload_library(const char *libname); static bool file_exists(const char *name); static char *expand_dynamic_library_name(const char *name); From 6fb12cbcd6f1a971e113d7d32cc7ee920db4e097 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 10 Oct 2016 12:22:58 +0300 Subject: [PATCH 300/871] Remove some unnecessary #includes. Amit Langote --- src/backend/commands/aggregatecmds.c | 1 - src/backend/commands/collationcmds.c | 1 - src/backend/commands/conversioncmds.c | 1 - src/backend/commands/lockcmds.c | 1 - 4 files changed, 4 deletions(-) diff --git a/src/backend/commands/aggregatecmds.c b/src/backend/commands/aggregatecmds.c index b36f09eb7b..19db38d7fc 100644 --- a/src/backend/commands/aggregatecmds.c +++ b/src/backend/commands/aggregatecmds.c @@ -22,7 +22,6 @@ */ #include "postgres.h" -#include "access/heapam.h" #include "access/htup_details.h" #include "catalog/dependency.h" #include "catalog/indexing.h" diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c index 0c75d16f36..9bba748708 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -14,7 +14,6 @@ */ #include "postgres.h" -#include "access/heapam.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/dependency.h" diff --git a/src/backend/commands/conversioncmds.c b/src/backend/commands/conversioncmds.c index 175d4ab685..2c85d26f4d 100644 --- a/src/backend/commands/conversioncmds.c +++ b/src/backend/commands/conversioncmds.c @@ -14,7 +14,6 @@ */ #include "postgres.h" -#include "access/heapam.h" #include "access/htup_details.h" #include "catalog/dependency.h" #include "catalog/indexing.h" diff --git a/src/backend/commands/lockcmds.c b/src/backend/commands/lockcmds.c index 175d1f3f2e..a0c0d75977 100644 --- a/src/backend/commands/lockcmds.c +++ b/src/backend/commands/lockcmds.c @@ -14,7 +14,6 @@ */ #include "postgres.h" -#include "access/heapam.h" #include "catalog/namespace.h" #include "catalog/pg_inherits_fn.h" #include "commands/lockcmds.h" From 886f6c5ccdb500eeeec7e0abdf1500e20a304c45 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 10 Oct 2016 10:35:58 -0400 Subject: [PATCH 301/871] In PQsendQueryStart(), avoid leaking any left-over async result. Ordinarily there would not be an async result sitting around at this point, but it appears that in corner cases there can be. Considering all the work we're about to launch, it's hardly going to cost anything noticeable to check. It's been like this forever, so back-patch to all supported branches. Report: --- src/interfaces/libpq/fe-exec.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c index a9ba54628f..87ff5659ff 100644 --- a/src/interfaces/libpq/fe-exec.c +++ b/src/interfaces/libpq/fe-exec.c @@ -1386,8 +1386,7 @@ PQsendQueryStart(PGconn *conn) } /* initialize async result-accumulation state */ - conn->result = NULL; - conn->next_result = NULL; + pqClearAsyncResult(conn); /* reset single-row processing mode */ conn->singleRowMode = false; From 0137caf273f4297c4d36df3a542d7c0c853e75be Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Mon, 10 Oct 2016 13:41:57 -0700 Subject: [PATCH 302/871] Make regression tests less dependent on hash table order. Upcoming changes to the hash table code used, among others, for grouping and set operations will change the output order for a few queries. To make it less likely that actual bugs are hidden between regression test ordering changes, and to make the tests robust against platform dependant ordering, add ORDER BYs guaranteeing the output order. As it's possible that some of the changes expose platform dependant ordering, push this earlier, to let the buildfarm shake out potentially unstable results. Discussion: <20160727004333.r3e2k2y6fvk2ntup@alap3.anarazel.de> --- src/test/regress/expected/matview.out | 6 +- src/test/regress/expected/psql.out | 2 +- src/test/regress/expected/tsrf.out | 8 +- src/test/regress/expected/union.out | 129 ++++++++++++++------------ src/test/regress/expected/with.out | 8 +- src/test/regress/sql/matview.sql | 4 +- src/test/regress/sql/psql.sql | 2 +- src/test/regress/sql/tsrf.sql | 2 +- src/test/regress/sql/union.sql | 63 +++++++------ src/test/regress/sql/with.sql | 8 +- 10 files changed, 123 insertions(+), 109 deletions(-) diff --git a/src/test/regress/expected/matview.out b/src/test/regress/expected/matview.out index e7d0ad1d86..08cffcfae7 100644 --- a/src/test/regress/expected/matview.out +++ b/src/test/regress/expected/matview.out @@ -33,7 +33,7 @@ SELECT relispopulated FROM pg_class WHERE oid = 'mvtest_tm'::regclass; f (1 row) -SELECT * FROM mvtest_tm; +SELECT * FROM mvtest_tm ORDER BY type; ERROR: materialized view "mvtest_tm" has not been populated HINT: Use the REFRESH MATERIALIZED VIEW command. REFRESH MATERIALIZED VIEW mvtest_tm; @@ -44,12 +44,12 @@ SELECT relispopulated FROM pg_class WHERE oid = 'mvtest_tm'::regclass; (1 row) CREATE UNIQUE INDEX mvtest_tm_type ON mvtest_tm (type); -SELECT * FROM mvtest_tm; +SELECT * FROM mvtest_tm ORDER BY type; type | totamt ------+-------- + x | 5 y | 12 z | 11 - x | 5 (3 rows) -- create various views diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index 017b79ea9c..464436ab3b 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -123,7 +123,7 @@ unicode_header_linestyle single prepare q as select array_to_string(array_agg(repeat('x',2*n)),E'\n') as "ab c", array_to_string(array_agg(repeat('y',20-2*n)),E'\n') as "a -bc" from generate_series(1,10) as n(n) group by n>1 ; +bc" from generate_series(1,10) as n(n) group by n>1 order by n>1; \pset linestyle ascii \pset expanded off \pset columns 40 diff --git a/src/test/regress/expected/tsrf.out b/src/test/regress/expected/tsrf.out index d9a5f137dc..8c54f717df 100644 --- a/src/test/regress/expected/tsrf.out +++ b/src/test/regress/expected/tsrf.out @@ -187,15 +187,15 @@ SELECT SUM(count(*)) OVER(PARTITION BY generate_series(1,3) ORDER BY generate_se (3 rows) -- sorting + grouping -SELECT few.dataa, count(*), min(id), max(id), generate_series(1,3) FROM few GROUP BY few.dataa ORDER BY 5; +SELECT few.dataa, count(*), min(id), max(id), generate_series(1,3) FROM few GROUP BY few.dataa ORDER BY 5, 1; dataa | count | min | max | generate_series -------+-------+-----+-----+----------------- - b | 1 | 3 | 3 | 1 a | 2 | 1 | 2 | 1 - b | 1 | 3 | 3 | 2 + b | 1 | 3 | 3 | 1 a | 2 | 1 | 2 | 2 - b | 1 | 3 | 3 | 3 + b | 1 | 3 | 3 | 2 a | 2 | 1 | 2 | 3 + b | 1 | 3 | 3 | 3 (6 rows) -- grouping sets are a bit special, they produce NULLs in columns not actually NULL diff --git a/src/test/regress/expected/union.out b/src/test/regress/expected/union.out index 016571bd4a..67f5fc4361 100644 --- a/src/test/regress/expected/union.out +++ b/src/test/regress/expected/union.out @@ -2,14 +2,14 @@ -- UNION (also INTERSECT, EXCEPT) -- -- Simple UNION constructs -SELECT 1 AS two UNION SELECT 2; +SELECT 1 AS two UNION SELECT 2 ORDER BY 1; two ----- 1 2 (2 rows) -SELECT 1 AS one UNION SELECT 1; +SELECT 1 AS one UNION SELECT 1 ORDER BY 1; one ----- 1 @@ -29,7 +29,7 @@ SELECT 1 AS two UNION ALL SELECT 1; 1 (2 rows) -SELECT 1 AS three UNION SELECT 2 UNION SELECT 3; +SELECT 1 AS three UNION SELECT 2 UNION SELECT 3 ORDER BY 1; three ------- 1 @@ -37,14 +37,14 @@ SELECT 1 AS three UNION SELECT 2 UNION SELECT 3; 3 (3 rows) -SELECT 1 AS two UNION SELECT 2 UNION SELECT 2; +SELECT 1 AS two UNION SELECT 2 UNION SELECT 2 ORDER BY 1; two ----- 1 2 (2 rows) -SELECT 1 AS three UNION SELECT 2 UNION ALL SELECT 2; +SELECT 1 AS three UNION SELECT 2 UNION ALL SELECT 2 ORDER BY 1; three ------- 1 @@ -52,7 +52,7 @@ SELECT 1 AS three UNION SELECT 2 UNION ALL SELECT 2; 2 (3 rows) -SELECT 1.1 AS two UNION SELECT 2.2; +SELECT 1.1 AS two UNION SELECT 2.2 ORDER BY 1; two ----- 1.1 @@ -60,41 +60,41 @@ SELECT 1.1 AS two UNION SELECT 2.2; (2 rows) -- Mixed types -SELECT 1.1 AS two UNION SELECT 2; +SELECT 1.1 AS two UNION SELECT 2 ORDER BY 1; two ----- 1.1 2 (2 rows) -SELECT 1 AS two UNION SELECT 2.2; +SELECT 1 AS two UNION SELECT 2.2 ORDER BY 1; two ----- 1 2.2 (2 rows) -SELECT 1 AS one UNION SELECT 1.0::float8; +SELECT 1 AS one UNION SELECT 1.0::float8 ORDER BY 1; one ----- 1 (1 row) -SELECT 1.1 AS two UNION ALL SELECT 2; +SELECT 1.1 AS two UNION ALL SELECT 2 ORDER BY 1; two ----- 1.1 2 (2 rows) -SELECT 1.0::float8 AS two UNION ALL SELECT 1; +SELECT 1.0::float8 AS two UNION ALL SELECT 1 ORDER BY 1; two ----- 1 1 (2 rows) -SELECT 1.1 AS three UNION SELECT 2 UNION SELECT 3; +SELECT 1.1 AS three UNION SELECT 2 UNION SELECT 3 ORDER BY 1; three ------- 1.1 @@ -109,7 +109,7 @@ SELECT 1.1::float8 AS two UNION SELECT 2 UNION SELECT 2.0::float8 ORDER BY 1; 2 (2 rows) -SELECT 1.1 AS three UNION SELECT 2 UNION ALL SELECT 2; +SELECT 1.1 AS three UNION SELECT 2 UNION ALL SELECT 2 ORDER BY 1; three ------- 1.1 @@ -117,7 +117,7 @@ SELECT 1.1 AS three UNION SELECT 2 UNION ALL SELECT 2; 2 (3 rows) -SELECT 1.1 AS two UNION (SELECT 2 UNION ALL SELECT 2); +SELECT 1.1 AS two UNION (SELECT 2 UNION ALL SELECT 2) ORDER BY 1; two ----- 1.1 @@ -195,7 +195,8 @@ SELECT f1 AS five FROM FLOAT8_TBL WHERE f1 BETWEEN -1e6 AND 1e6 UNION SELECT f1 FROM INT4_TBL - WHERE f1 BETWEEN 0 AND 1000000; + WHERE f1 BETWEEN 0 AND 1000000 +ORDER BY 1; five ----------------------- -1004.3 @@ -260,19 +261,19 @@ ORDER BY 1; -- -- INTERSECT and EXCEPT -- -SELECT q2 FROM int8_tbl INTERSECT SELECT q1 FROM int8_tbl; +SELECT q2 FROM int8_tbl INTERSECT SELECT q1 FROM int8_tbl ORDER BY 1; q2 ------------------ - 4567890123456789 123 + 4567890123456789 (2 rows) -SELECT q2 FROM int8_tbl INTERSECT ALL SELECT q1 FROM int8_tbl; +SELECT q2 FROM int8_tbl INTERSECT ALL SELECT q1 FROM int8_tbl ORDER BY 1; q2 ------------------ + 123 4567890123456789 4567890123456789 - 123 (3 rows) SELECT q2 FROM int8_tbl EXCEPT SELECT q1 FROM int8_tbl ORDER BY 1; @@ -297,24 +298,24 @@ SELECT q2 FROM int8_tbl EXCEPT ALL SELECT DISTINCT q1 FROM int8_tbl ORDER BY 1; 4567890123456789 (3 rows) -SELECT q1 FROM int8_tbl EXCEPT SELECT q2 FROM int8_tbl; +SELECT q1 FROM int8_tbl EXCEPT SELECT q2 FROM int8_tbl ORDER BY 1; q1 ---- (0 rows) -SELECT q1 FROM int8_tbl EXCEPT ALL SELECT q2 FROM int8_tbl; +SELECT q1 FROM int8_tbl EXCEPT ALL SELECT q2 FROM int8_tbl ORDER BY 1; q1 ------------------ - 4567890123456789 123 + 4567890123456789 (2 rows) -SELECT q1 FROM int8_tbl EXCEPT ALL SELECT DISTINCT q2 FROM int8_tbl; +SELECT q1 FROM int8_tbl EXCEPT ALL SELECT DISTINCT q2 FROM int8_tbl ORDER BY 1; q1 ------------------ + 123 4567890123456789 4567890123456789 - 123 (3 rows) SELECT q1 FROM int8_tbl EXCEPT ALL SELECT q1 FROM int8_tbl FOR NO KEY UPDATE; @@ -322,7 +323,7 @@ ERROR: FOR NO KEY UPDATE is not allowed with UNION/INTERSECT/EXCEPT -- -- Mixed types -- -SELECT f1 FROM float8_tbl INTERSECT SELECT f1 FROM int4_tbl; +SELECT f1 FROM float8_tbl INTERSECT SELECT f1 FROM int4_tbl ORDER BY 1; f1 ---- 0 @@ -340,30 +341,30 @@ SELECT f1 FROM float8_tbl EXCEPT SELECT f1 FROM int4_tbl ORDER BY 1; -- -- Operator precedence and (((((extra))))) parentheses -- -SELECT q1 FROM int8_tbl INTERSECT SELECT q2 FROM int8_tbl UNION ALL SELECT q2 FROM int8_tbl; +SELECT q1 FROM int8_tbl INTERSECT SELECT q2 FROM int8_tbl UNION ALL SELECT q2 FROM int8_tbl ORDER BY 1; q1 ------------------- - 4567890123456789 + -4567890123456789 + 123 123 456 4567890123456789 - 123 4567890123456789 - -4567890123456789 + 4567890123456789 (7 rows) -SELECT q1 FROM int8_tbl INTERSECT (((SELECT q2 FROM int8_tbl UNION ALL SELECT q2 FROM int8_tbl))); +SELECT q1 FROM int8_tbl INTERSECT (((SELECT q2 FROM int8_tbl UNION ALL SELECT q2 FROM int8_tbl))) ORDER BY 1; q1 ------------------ - 4567890123456789 123 + 4567890123456789 (2 rows) -(((SELECT q1 FROM int8_tbl INTERSECT SELECT q2 FROM int8_tbl))) UNION ALL SELECT q2 FROM int8_tbl; +(((SELECT q1 FROM int8_tbl INTERSECT SELECT q2 FROM int8_tbl ORDER BY 1))) UNION ALL SELECT q2 FROM int8_tbl; q1 ------------------- - 4567890123456789 123 + 4567890123456789 456 4567890123456789 123 @@ -416,11 +417,11 @@ LINE 1: ... int8_tbl EXCEPT SELECT q2 FROM int8_tbl ORDER BY q2 LIMIT 1... ^ HINT: There is a column named "q2" in table "*SELECT* 2", but it cannot be referenced from this part of the query. -- But this should work: -SELECT q1 FROM int8_tbl EXCEPT (((SELECT q2 FROM int8_tbl ORDER BY q2 LIMIT 1))); +SELECT q1 FROM int8_tbl EXCEPT (((SELECT q2 FROM int8_tbl ORDER BY q2 LIMIT 1))) ORDER BY 1; q1 ------------------ - 4567890123456789 123 + 4567890123456789 (2 rows) -- @@ -593,23 +594,27 @@ SELECT * FROM (SELECT 1 AS t, 2 AS x UNION SELECT 2 AS t, 4 AS x) ss -WHERE x < 4; - QUERY PLAN --------------------------------------------- - Unique - -> Sort - Sort Key: (1), (2) - -> Append - -> Result - -> Result - One-Time Filter: false -(7 rows) +WHERE x < 4 +ORDER BY x; + QUERY PLAN +-------------------------------------------------- + Sort + Sort Key: (2) + -> Unique + -> Sort + Sort Key: (1), (2) + -> Append + -> Result + -> Result + One-Time Filter: false +(9 rows) SELECT * FROM (SELECT 1 AS t, 2 AS x UNION SELECT 2 AS t, 4 AS x) ss -WHERE x < 4; +WHERE x < 4 +ORDER BY x; t | x ---+--- 1 | 2 @@ -653,24 +658,28 @@ SELECT * FROM (SELECT 1 AS t, (random()*3)::int AS x UNION SELECT 2 AS t, 4 AS x) ss -WHERE x > 3; - QUERY PLAN ------------------------------------------------------------------------------- - Subquery Scan on ss - Filter: (ss.x > 3) - -> Unique - -> Sort - Sort Key: (1), (((random() * '3'::double precision))::integer) - -> Append - -> Result - -> Result -(8 rows) +WHERE x > 3 +ORDER BY x; + QUERY PLAN +------------------------------------------------------------------------------------ + Sort + Sort Key: ss.x + -> Subquery Scan on ss + Filter: (ss.x > 3) + -> Unique + -> Sort + Sort Key: (1), (((random() * '3'::double precision))::integer) + -> Append + -> Result + -> Result +(10 rows) SELECT * FROM (SELECT 1 AS t, (random()*3)::int AS x UNION SELECT 2 AS t, 4 AS x) ss -WHERE x > 3; +WHERE x > 3 +ORDER BY x; t | x ---+--- 2 | 4 diff --git a/src/test/regress/expected/with.out b/src/test/regress/expected/with.out index 137420d9b7..1b7f57b4de 100644 --- a/src/test/regress/expected/with.out +++ b/src/test/regress/expected/with.out @@ -1244,7 +1244,7 @@ WITH outermost(x) AS ( SELECT * FROM innermost UNION SELECT 3) ) -SELECT * FROM outermost; +SELECT * FROM outermost ORDER BY 1; x --- 1 @@ -1258,7 +1258,7 @@ WITH outermost(x) AS ( SELECT * FROM outermost -- fail UNION SELECT * FROM innermost) ) -SELECT * FROM outermost; +SELECT * FROM outermost ORDER BY 1; ERROR: relation "outermost" does not exist LINE 4: SELECT * FROM outermost ^ @@ -1270,7 +1270,7 @@ WITH RECURSIVE outermost(x) AS ( SELECT * FROM outermost UNION SELECT * FROM innermost) ) -SELECT * FROM outermost; +SELECT * FROM outermost ORDER BY 1; x --- 1 @@ -1282,7 +1282,7 @@ WITH RECURSIVE outermost(x) AS ( SELECT * FROM innermost UNION SELECT * from outermost ) -SELECT * FROM outermost; +SELECT * FROM outermost ORDER BY 1; ERROR: recursive reference to query "outermost" must not appear within a subquery LINE 2: WITH innermost as (SELECT 2 FROM outermost) ^ diff --git a/src/test/regress/sql/matview.sql b/src/test/regress/sql/matview.sql index 5f3269def8..65a743ced9 100644 --- a/src/test/regress/sql/matview.sql +++ b/src/test/regress/sql/matview.sql @@ -16,11 +16,11 @@ EXPLAIN (costs off) CREATE MATERIALIZED VIEW mvtest_tm AS SELECT type, sum(amt) AS totamt FROM mvtest_t GROUP BY type WITH NO DATA; CREATE MATERIALIZED VIEW mvtest_tm AS SELECT type, sum(amt) AS totamt FROM mvtest_t GROUP BY type WITH NO DATA; SELECT relispopulated FROM pg_class WHERE oid = 'mvtest_tm'::regclass; -SELECT * FROM mvtest_tm; +SELECT * FROM mvtest_tm ORDER BY type; REFRESH MATERIALIZED VIEW mvtest_tm; SELECT relispopulated FROM pg_class WHERE oid = 'mvtest_tm'::regclass; CREATE UNIQUE INDEX mvtest_tm_type ON mvtest_tm (type); -SELECT * FROM mvtest_tm; +SELECT * FROM mvtest_tm ORDER BY type; -- create various views EXPLAIN (costs off) diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql index 4dc0745f1d..900aa7ee1e 100644 --- a/src/test/regress/sql/psql.sql +++ b/src/test/regress/sql/psql.sql @@ -67,7 +67,7 @@ select 'drop table gexec_test', 'select ''2000-01-01''::date as party_over' prepare q as select array_to_string(array_agg(repeat('x',2*n)),E'\n') as "ab c", array_to_string(array_agg(repeat('y',20-2*n)),E'\n') as "a -bc" from generate_series(1,10) as n(n) group by n>1 ; +bc" from generate_series(1,10) as n(n) group by n>1 order by n>1; \pset linestyle ascii diff --git a/src/test/regress/sql/tsrf.sql b/src/test/regress/sql/tsrf.sql index 4f854c8b83..cf2fbe3e52 100644 --- a/src/test/regress/sql/tsrf.sql +++ b/src/test/regress/sql/tsrf.sql @@ -56,7 +56,7 @@ SELECT id,lag(id) OVER(), count(*) OVER(), generate_series(1,3) FROM few; SELECT SUM(count(*)) OVER(PARTITION BY generate_series(1,3) ORDER BY generate_series(1,3)), generate_series(1,3) g FROM few GROUP BY g; -- sorting + grouping -SELECT few.dataa, count(*), min(id), max(id), generate_series(1,3) FROM few GROUP BY few.dataa ORDER BY 5; +SELECT few.dataa, count(*), min(id), max(id), generate_series(1,3) FROM few GROUP BY few.dataa ORDER BY 5, 1; -- grouping sets are a bit special, they produce NULLs in columns not actually NULL SELECT dataa, datab b, generate_series(1,2) g, count(*) FROM few GROUP BY CUBE(dataa, datab); diff --git a/src/test/regress/sql/union.sql b/src/test/regress/sql/union.sql index 9ff1551e5d..debd99ed51 100644 --- a/src/test/regress/sql/union.sql +++ b/src/test/regress/sql/union.sql @@ -4,41 +4,41 @@ -- Simple UNION constructs -SELECT 1 AS two UNION SELECT 2; +SELECT 1 AS two UNION SELECT 2 ORDER BY 1; -SELECT 1 AS one UNION SELECT 1; +SELECT 1 AS one UNION SELECT 1 ORDER BY 1; SELECT 1 AS two UNION ALL SELECT 2; SELECT 1 AS two UNION ALL SELECT 1; -SELECT 1 AS three UNION SELECT 2 UNION SELECT 3; +SELECT 1 AS three UNION SELECT 2 UNION SELECT 3 ORDER BY 1; -SELECT 1 AS two UNION SELECT 2 UNION SELECT 2; +SELECT 1 AS two UNION SELECT 2 UNION SELECT 2 ORDER BY 1; -SELECT 1 AS three UNION SELECT 2 UNION ALL SELECT 2; +SELECT 1 AS three UNION SELECT 2 UNION ALL SELECT 2 ORDER BY 1; -SELECT 1.1 AS two UNION SELECT 2.2; +SELECT 1.1 AS two UNION SELECT 2.2 ORDER BY 1; -- Mixed types -SELECT 1.1 AS two UNION SELECT 2; +SELECT 1.1 AS two UNION SELECT 2 ORDER BY 1; -SELECT 1 AS two UNION SELECT 2.2; +SELECT 1 AS two UNION SELECT 2.2 ORDER BY 1; -SELECT 1 AS one UNION SELECT 1.0::float8; +SELECT 1 AS one UNION SELECT 1.0::float8 ORDER BY 1; -SELECT 1.1 AS two UNION ALL SELECT 2; +SELECT 1.1 AS two UNION ALL SELECT 2 ORDER BY 1; -SELECT 1.0::float8 AS two UNION ALL SELECT 1; +SELECT 1.0::float8 AS two UNION ALL SELECT 1 ORDER BY 1; -SELECT 1.1 AS three UNION SELECT 2 UNION SELECT 3; +SELECT 1.1 AS three UNION SELECT 2 UNION SELECT 3 ORDER BY 1; SELECT 1.1::float8 AS two UNION SELECT 2 UNION SELECT 2.0::float8 ORDER BY 1; -SELECT 1.1 AS three UNION SELECT 2 UNION ALL SELECT 2; +SELECT 1.1 AS three UNION SELECT 2 UNION ALL SELECT 2 ORDER BY 1; -SELECT 1.1 AS two UNION (SELECT 2 UNION ALL SELECT 2); +SELECT 1.1 AS two UNION (SELECT 2 UNION ALL SELECT 2) ORDER BY 1; -- -- Try testing from tables... @@ -66,7 +66,8 @@ SELECT f1 AS five FROM FLOAT8_TBL WHERE f1 BETWEEN -1e6 AND 1e6 UNION SELECT f1 FROM INT4_TBL - WHERE f1 BETWEEN 0 AND 1000000; + WHERE f1 BETWEEN 0 AND 1000000 +ORDER BY 1; SELECT CAST(f1 AS char(4)) AS three FROM VARCHAR_TBL UNION @@ -93,9 +94,9 @@ ORDER BY 1; -- INTERSECT and EXCEPT -- -SELECT q2 FROM int8_tbl INTERSECT SELECT q1 FROM int8_tbl; +SELECT q2 FROM int8_tbl INTERSECT SELECT q1 FROM int8_tbl ORDER BY 1; -SELECT q2 FROM int8_tbl INTERSECT ALL SELECT q1 FROM int8_tbl; +SELECT q2 FROM int8_tbl INTERSECT ALL SELECT q1 FROM int8_tbl ORDER BY 1; SELECT q2 FROM int8_tbl EXCEPT SELECT q1 FROM int8_tbl ORDER BY 1; @@ -103,11 +104,11 @@ SELECT q2 FROM int8_tbl EXCEPT ALL SELECT q1 FROM int8_tbl ORDER BY 1; SELECT q2 FROM int8_tbl EXCEPT ALL SELECT DISTINCT q1 FROM int8_tbl ORDER BY 1; -SELECT q1 FROM int8_tbl EXCEPT SELECT q2 FROM int8_tbl; +SELECT q1 FROM int8_tbl EXCEPT SELECT q2 FROM int8_tbl ORDER BY 1; -SELECT q1 FROM int8_tbl EXCEPT ALL SELECT q2 FROM int8_tbl; +SELECT q1 FROM int8_tbl EXCEPT ALL SELECT q2 FROM int8_tbl ORDER BY 1; -SELECT q1 FROM int8_tbl EXCEPT ALL SELECT DISTINCT q2 FROM int8_tbl; +SELECT q1 FROM int8_tbl EXCEPT ALL SELECT DISTINCT q2 FROM int8_tbl ORDER BY 1; SELECT q1 FROM int8_tbl EXCEPT ALL SELECT q1 FROM int8_tbl FOR NO KEY UPDATE; @@ -115,7 +116,7 @@ SELECT q1 FROM int8_tbl EXCEPT ALL SELECT q1 FROM int8_tbl FOR NO KEY UPDATE; -- Mixed types -- -SELECT f1 FROM float8_tbl INTERSECT SELECT f1 FROM int4_tbl; +SELECT f1 FROM float8_tbl INTERSECT SELECT f1 FROM int4_tbl ORDER BY 1; SELECT f1 FROM float8_tbl EXCEPT SELECT f1 FROM int4_tbl ORDER BY 1; @@ -123,11 +124,11 @@ SELECT f1 FROM float8_tbl EXCEPT SELECT f1 FROM int4_tbl ORDER BY 1; -- Operator precedence and (((((extra))))) parentheses -- -SELECT q1 FROM int8_tbl INTERSECT SELECT q2 FROM int8_tbl UNION ALL SELECT q2 FROM int8_tbl; +SELECT q1 FROM int8_tbl INTERSECT SELECT q2 FROM int8_tbl UNION ALL SELECT q2 FROM int8_tbl ORDER BY 1; -SELECT q1 FROM int8_tbl INTERSECT (((SELECT q2 FROM int8_tbl UNION ALL SELECT q2 FROM int8_tbl))); +SELECT q1 FROM int8_tbl INTERSECT (((SELECT q2 FROM int8_tbl UNION ALL SELECT q2 FROM int8_tbl))) ORDER BY 1; -(((SELECT q1 FROM int8_tbl INTERSECT SELECT q2 FROM int8_tbl))) UNION ALL SELECT q2 FROM int8_tbl; +(((SELECT q1 FROM int8_tbl INTERSECT SELECT q2 FROM int8_tbl ORDER BY 1))) UNION ALL SELECT q2 FROM int8_tbl; SELECT q1 FROM int8_tbl UNION ALL SELECT q2 FROM int8_tbl EXCEPT SELECT q1 FROM int8_tbl ORDER BY 1; @@ -147,7 +148,7 @@ ORDER BY q2,q1; SELECT q1 FROM int8_tbl EXCEPT SELECT q2 FROM int8_tbl ORDER BY q2 LIMIT 1; -- But this should work: -SELECT q1 FROM int8_tbl EXCEPT (((SELECT q2 FROM int8_tbl ORDER BY q2 LIMIT 1))); +SELECT q1 FROM int8_tbl EXCEPT (((SELECT q2 FROM int8_tbl ORDER BY q2 LIMIT 1))) ORDER BY 1; -- -- New syntaxes (7.1) permit new tests @@ -261,13 +262,15 @@ SELECT * FROM (SELECT 1 AS t, 2 AS x UNION SELECT 2 AS t, 4 AS x) ss -WHERE x < 4; +WHERE x < 4 +ORDER BY x; SELECT * FROM (SELECT 1 AS t, 2 AS x UNION SELECT 2 AS t, 4 AS x) ss -WHERE x < 4; +WHERE x < 4 +ORDER BY x; explain (costs off) SELECT * FROM @@ -289,13 +292,15 @@ SELECT * FROM (SELECT 1 AS t, (random()*3)::int AS x UNION SELECT 2 AS t, 4 AS x) ss -WHERE x > 3; +WHERE x > 3 +ORDER BY x; SELECT * FROM (SELECT 1 AS t, (random()*3)::int AS x UNION SELECT 2 AS t, 4 AS x) ss -WHERE x > 3; +WHERE x > 3 +ORDER BY x; -- Test proper handling of parameterized appendrel paths when the -- potential join qual is expensive diff --git a/src/test/regress/sql/with.sql b/src/test/regress/sql/with.sql index 133ff0b195..7ee32bab8f 100644 --- a/src/test/regress/sql/with.sql +++ b/src/test/regress/sql/with.sql @@ -575,7 +575,7 @@ WITH outermost(x) AS ( SELECT * FROM innermost UNION SELECT 3) ) -SELECT * FROM outermost; +SELECT * FROM outermost ORDER BY 1; WITH outermost(x) AS ( SELECT 1 @@ -583,7 +583,7 @@ WITH outermost(x) AS ( SELECT * FROM outermost -- fail UNION SELECT * FROM innermost) ) -SELECT * FROM outermost; +SELECT * FROM outermost ORDER BY 1; WITH RECURSIVE outermost(x) AS ( SELECT 1 @@ -591,14 +591,14 @@ WITH RECURSIVE outermost(x) AS ( SELECT * FROM outermost UNION SELECT * FROM innermost) ) -SELECT * FROM outermost; +SELECT * FROM outermost ORDER BY 1; WITH RECURSIVE outermost(x) AS ( WITH innermost as (SELECT 2 FROM outermost) -- fail SELECT * FROM innermost UNION SELECT * from outermost ) -SELECT * FROM outermost; +SELECT * FROM outermost ORDER BY 1; -- -- This test will fail with the old implementation of PARAM_EXEC parameter From 3d21f08bccd316c3850a1943c1ee1e381dab1588 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 10 Oct 2016 15:11:33 -0400 Subject: [PATCH 303/871] Update user docs for switch to POSIX semaphores. Since commit ecb0d20a9 hasn't crashed and burned, here's the promised docs update for it. In addition to explaining that Linux and FreeBSD ports now use POSIX semaphores, I did some wordsmithing on pre-existing wording; in particular trying to clarify which SysV parameters need to be set with an eye to total usage across all applications. --- doc/src/sgml/runtime.sgml | 83 +++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 88ec120841..a8efb3d006 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -605,27 +605,47 @@ psql: could not connect to server: No such file or directory - Shared memory and semaphores are collectively referred to as - System V - IPC (together with message queues, which are not - relevant for PostgreSQL). Except on - Windows, where PostgreSQL - provides its own replacement implementation of these facilities, these - facilities are required in order to run - PostgreSQL. + PostgreSQL requires the operating system to provide + inter-process communication (IPC) features, specifically + shared memory and semaphores. Unix-derived systems typically provide + System V IPC, + POSIX IPC, or both. + Windows has its own implementation of + these features and is not discussed here. The complete lack of these facilities is usually manifested by an - Illegal system call error upon server start. In - that case there is no alternative but to reconfigure your + Illegal system call error upon server + start. In that case there is no alternative but to reconfigure your kernel. PostgreSQL won't work without them. This situation is rare, however, among modern operating systems. - When PostgreSQL exceeds one of the various hard - IPC limits, the server will refuse to start and + Upon starting the server, PostgreSQL normally allocates + a very small amount of System V shared memory, as well as a much larger + amount of POSIX (mmap) shared memory. + In addition a significant number of semaphores, which can be either + System V or POSIX style, are created at server startup. Currently, + POSIX semaphores are used on Linux and FreeBSD systems while other + platforms use System V semaphores. + + + + + Prior to PostgreSQL 9.3, only System V shared memory + was used, so the amount of System V shared memory required to start the + server was much larger. If you are running an older version of the + server, please consult the documentation for your server version. + + + + + System V IPC features are typically constrained by + system-wide allocation limits. + When PostgreSQL exceeds one of these limits, + the server will refuse to start and should leave an instructive error message describing the problem and what to do about it. (See also .) The relevant kernel @@ -634,15 +654,6 @@ psql: could not connect to server: No such file or directory them, however, vary. Suggestions for some platforms are given below. - - - Prior to PostgreSQL 9.3, the amount of System V shared - memory required to start the server was much larger. If you are running - an older version of the server, please consult the documentation for - your server version. - - - <systemitem class="osname">System V</> <acronym>IPC</> Parameters @@ -651,7 +662,7 @@ psql: could not connect to server: No such file or directory Name Description - Reasonable values + Values needed to run one PostgreSQL instance @@ -659,7 +670,7 @@ psql: could not connect to server: No such file or directory SHMMAX Maximum size of shared memory segment (bytes) - at least 1kB (more if running many copies of the server) + at least 1kB, but the default is usually much higher @@ -671,7 +682,9 @@ psql: could not connect to server: No such file or directory SHMALL Total amount of shared memory available (bytes or pages) - if bytes, same as SHMMAX; if pages, ceil(SHMMAX/PAGE_SIZE) + same as SHMMAX if bytes, + or ceil(SHMMAX/PAGE_SIZE) if pages, + plus room for other applications @@ -689,7 +702,7 @@ psql: could not connect to server: No such file or directory SEMMNI Maximum number of semaphore identifiers (i.e., sets) - at least ceil((max_connections + autovacuum_max_workers + max_worker_processes + 5) / 16) + at least ceil((max_connections + autovacuum_max_workers + max_worker_processes + 5) / 16) plus room for other applications @@ -725,9 +738,8 @@ psql: could not connect to server: No such file or directory (typically 48 bytes, on 64-bit platforms) for each copy of the server. On most modern operating systems, this amount can easily be allocated. However, if you are running many copies of the server, or if other - applications are also using System V shared memory, it may be necessary - to increase SHMMAX, the maximum size in bytes of a shared - memory segment, or SHMALL, the total amount of System V shared + applications are also using System V shared memory, it may be necessary to + increase SHMALL, which is the total amount of System V shared memory system-wide. Note that SHMALL is measured in pages rather than bytes on many systems. @@ -742,6 +754,7 @@ psql: could not connect to server: No such file or directory + When using System V semaphores, PostgreSQL uses one semaphore per allowed connection (), allowed autovacuum worker process () and allowed background @@ -779,15 +792,19 @@ psql: could not connect to server: No such file or directory - The SEMMSL parameter, which determines how many - semaphores can be in a set, must be at least 17 for + Various other settings related to semaphore undo, such as + SEMMNU and SEMUME, do not affect PostgreSQL. - Various other settings related to semaphore undo, such as - SEMMNU and SEMUME, do not affect - PostgreSQL. + When using POSIX semaphores, the number of semaphores needed is the + same as for System V, that is one semaphore per allowed connection + (), allowed autovacuum worker process + () and allowed background + process (). + On the platforms where this option is preferred, there is no specific + kernel limit on the number of POSIX semaphores. From e34318725ca5b274efd6f57ea7460e89f4dca9f9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 11 Oct 2016 10:08:45 -0400 Subject: [PATCH 304/871] Improve documentation for CREATE RECURSIVE VIEW. It was perhaps not entirely clear that internal self-references shouldn't be schema-qualified even if the view name is written with a schema. Spell it out. Discussion: <871sznz69m.fsf@metapensiero.it> --- doc/src/sgml/ref/create_view.sgml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/ref/create_view.sgml b/doc/src/sgml/ref/create_view.sgml index ede1698051..8641e1925e 100644 --- a/doc/src/sgml/ref/create_view.sgml +++ b/doc/src/sgml/ref/create_view.sgml @@ -87,13 +87,13 @@ CREATE [ OR REPLACE ] [ TEMP | TEMPORARY ] [ RECURSIVE ] VIEW Creates a recursive view. The syntax -CREATE RECURSIVE VIEW name (columns) AS SELECT ...; +CREATE RECURSIVE VIEW [ schema . ] view_name (column_names) AS SELECT ...; is equivalent to -CREATE VIEW name AS WITH RECURSIVE name (columns) AS (SELECT ...) SELECT columns FROM name; +CREATE VIEW [ schema . ] view_name AS WITH RECURSIVE view_name (column_names) AS (SELECT ...) SELECT column_names FROM view_name; - A view column list must be specified for a recursive view. + A view column name list must be specified for a recursive view. @@ -462,11 +462,16 @@ CREATE VIEW comedies AS Create a recursive view consisting of the numbers from 1 to 100: -CREATE RECURSIVE VIEW nums_1_100 (n) AS +CREATE RECURSIVE VIEW public.nums_1_100 (n) AS VALUES (1) UNION ALL SELECT n+1 FROM nums_1_100 WHERE n < 100; - + + Notice that although the recursive view's name is schema-qualified in this + CREATE, its internal self-reference is not schema-qualified. + This is because the implicitly-created CTE's name cannot be + schema-qualified. + From c7e56811fa38cbc39efd6bdd4bb45f2f0444803e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 11 Oct 2016 10:33:59 -0400 Subject: [PATCH 305/871] Docs: grammatical fix. Fix poor grammar introduced in 741ccd501. --- doc/src/sgml/client-auth.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index a0d97ffbac..960f5b5871 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -711,7 +711,7 @@ local db1,db2,@demodbs all md5 When using an external authentication system such as Ident or GSSAPI, the name of the operating system user that initiated the connection - might not be the same as the database user that is to be connect as. + might not be the same as the database user (role) that is to be used. In this case, a user name map can be applied to map the operating system user name to a database user. To use user name mapping, specify map=map-name From 2b860f52ed1b1784cdf3f03886805f5bf250ea74 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 11 Oct 2016 11:26:04 -0400 Subject: [PATCH 306/871] Remove "sco" and "unixware" ports. SCO OpenServer and SCO UnixWare are more or less dead platforms. We have never had a buildfarm member testing the "sco" port, and the last "unixware" member was last heard from in 2012, so it's fair to doubt that the code even compiles anymore on either one. Remove both ports. We can always undo this if someone shows up with an interest in maintaining and testing these platforms. Discussion: <17177.1476136994@sss.pgh.pa.us> --- configure | 37 ------ configure.in | 14 --- doc/src/sgml/Makefile | 6 +- doc/src/sgml/dfunc.sgml | 26 ---- doc/src/sgml/installation.sgml | 164 +------------------------- doc/src/sgml/runtime.sgml | 58 --------- src/Makefile.global.in | 1 - src/Makefile.shlib | 24 ---- src/backend/libpq/pqcomm.c | 10 -- src/backend/port/dynloader/sco.c | 7 -- src/backend/port/dynloader/sco.h | 46 -------- src/backend/port/dynloader/unixware.c | 7 -- src/backend/port/dynloader/unixware.h | 49 -------- src/include/port/sco.h | 7 -- src/include/port/unixware.h | 11 -- src/include/storage/s_lock.h | 23 ---- src/makefiles/Makefile.sco | 13 -- src/makefiles/Makefile.unixware | 35 ------ src/port/getrusage.c | 1 - src/template/sco | 1 - src/template/unixware | 41 ------- 21 files changed, 6 insertions(+), 575 deletions(-) delete mode 100644 src/backend/port/dynloader/sco.c delete mode 100644 src/backend/port/dynloader/sco.h delete mode 100644 src/backend/port/dynloader/unixware.c delete mode 100644 src/backend/port/dynloader/unixware.h delete mode 100644 src/include/port/sco.h delete mode 100644 src/include/port/unixware.h delete mode 100644 src/makefiles/Makefile.sco delete mode 100644 src/makefiles/Makefile.unixware delete mode 100644 src/template/sco delete mode 100644 src/template/unixware diff --git a/configure b/configure index 1d94256a9e..f4f2f8b7ce 100755 --- a/configure +++ b/configure @@ -694,7 +694,6 @@ STRIP_SHARED_LIB STRIP_STATIC_LIB STRIP RANLIB -ld_R_works with_gnu_ld LD LDFLAGS_SL @@ -2867,9 +2866,7 @@ dragonfly*) template=netbsd ;; mingw*) template=win32 ;; netbsd*) template=netbsd ;; openbsd*) template=openbsd ;; - sco*) template=sco ;; solaris*) template=solaris ;; - sysv5*) template=unixware ;; esac if test x"$template" = x"" ; then @@ -6382,40 +6379,6 @@ with_gnu_ld=$ac_cv_prog_gnu_ld -case $host_os in sysv5*) - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ld -R works" >&5 -$as_echo_n "checking whether ld -R works... " >&6; } -if ${pgac_cv_prog_ld_R+:} false; then : - $as_echo_n "(cached) " >&6 -else - - pgac_save_LDFLAGS=$LDFLAGS; LDFLAGS="$LDFLAGS -Wl,-R/usr/lib" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - pgac_cv_prog_ld_R=yes -else - pgac_cv_prog_ld_R=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - LDFLAGS=$pgac_save_LDFLAGS - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_prog_ld_R" >&5 -$as_echo "$pgac_cv_prog_ld_R" >&6; } - ld_R_works=$pgac_cv_prog_ld_R - -esac if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args. set dummy ${ac_tool_prefix}ranlib; ac_word=$2 diff --git a/configure.in b/configure.in index 9ace625029..9f7611caeb 100644 --- a/configure.in +++ b/configure.in @@ -69,9 +69,7 @@ dragonfly*) template=netbsd ;; mingw*) template=win32 ;; netbsd*) template=netbsd ;; openbsd*) template=openbsd ;; - sco*) template=sco ;; solaris*) template=solaris ;; - sysv5*) template=unixware ;; esac if test x"$template" = x"" ; then @@ -871,18 +869,6 @@ AC_ARG_VAR(LDFLAGS_SL, [extra linker flags for linking shared libraries only]) PGAC_PROG_LD AC_SUBST(LD) AC_SUBST(with_gnu_ld) -case $host_os in sysv5*) - AC_CACHE_CHECK([whether ld -R works], [pgac_cv_prog_ld_R], - [ - pgac_save_LDFLAGS=$LDFLAGS; LDFLAGS="$LDFLAGS -Wl,-R/usr/lib" - AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], - [pgac_cv_prog_ld_R=yes], - [pgac_cv_prog_ld_R=no]) - LDFLAGS=$pgac_save_LDFLAGS - ]) - ld_R_works=$pgac_cv_prog_ld_R - AC_SUBST(ld_R_works) -esac AC_PROG_RANLIB PGAC_CHECK_STRIP AC_CHECK_TOOL(AR, ar, ar) diff --git a/doc/src/sgml/Makefile b/doc/src/sgml/Makefile index 24b895f3c3..5df2f04dd6 100644 --- a/doc/src/sgml/Makefile +++ b/doc/src/sgml/Makefile @@ -326,11 +326,7 @@ check: postgres.sgml $(ALMOSTALLSGML) check-tabs ## Install ## -install: install-html - -ifneq ($(PORTNAME), sco) -install: install-man -endif +install: install-html install-man installdirs: $(MKDIR_P) '$(DESTDIR)$(htmldir)'/html $(addprefix '$(DESTDIR)$(mandir)'/man, 1 3 $(sqlmansectnum)) diff --git a/doc/src/sgml/dfunc.sgml b/doc/src/sgml/dfunc.sgml index ba2684cc3c..6a4b7d6e97 100644 --- a/doc/src/sgml/dfunc.sgml +++ b/doc/src/sgml/dfunc.sgml @@ -200,32 +200,6 @@ cc -G -o foo.so foo.o gcc -fpic -c foo.c gcc -G -o foo.so foo.o - - - - - - - - UnixWare - UnixWareshared library - - - - The compiler flag to create PIC is with the SCO compiler and - with GCC. To link shared libraries, - the compiler option is with the SCO compiler - and with - GCC. - -cc -K PIC -c foo.c -cc -G -o foo.so foo.o - - or - -gcc -fpic -c foo.c -gcc -shared -o foo.so foo.o diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 5ee28fcf85..883e575946 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -2000,8 +2000,8 @@ kill `cat /usr/local/pgsql/data/postmaster.pid` In general, PostgreSQL can be expected to work on these CPU architectures: x86, x86_64, IA64, PowerPC, - PowerPC 64, S/390, S/390x, Sparc, Sparc 64, ARM, MIPS, MIPSEL, M68K, - and PA-RISC. Code support exists for M32R and VAX, but these + PowerPC 64, S/390, S/390x, Sparc, Sparc 64, ARM, MIPS, MIPSEL, + and PA-RISC. Code support exists for M68K, M32R, and VAX, but these architectures are not known to have been tested recently. It is often possible to build on an unsupported CPU type by configuring with , but performance will be poor. @@ -2010,11 +2010,11 @@ kill `cat /usr/local/pgsql/data/postmaster.pid` PostgreSQL can be expected to work on these operating systems: Linux (all recent distributions), Windows (Win2000 SP4 and later), - FreeBSD, OpenBSD, NetBSD, macOS, AIX, HP/UX, Solaris, - and UnixWare. Other Unix-like systems may also work but are not currently + FreeBSD, OpenBSD, NetBSD, macOS, AIX, HP/UX, and Solaris. + Other Unix-like systems may also work but are not currently being tested. In most cases, all CPU architectures supported by a given operating system will work. Look in - the below to see if + below to see if there is information specific to your operating system, particularly if using an older system. @@ -2639,160 +2639,6 @@ PHSS_30849 s700_800 u2comp/be/plugin library Patch - - SCO OpenServer and SCO UnixWare - - - SCO - installation on - - - - UnixWare - installation on - - - - PostgreSQL can be built on SCO UnixWare 7 and SCO OpenServer 5. - On OpenServer, you can use either the OpenServer Development Kit - or the Universal Development Kit. However, some tweaking may be - needed, as described below. - - - - Skunkware - - - You should locate your copy of the SCO Skunkware CD. The - Skunkware CD is included with UnixWare 7 and current versions of - OpenServer 5. Skunkware includes ready-to-install versions of - many popular programs that are available on the Internet. For - example, gzip, gunzip, GNU Make, Flex, and Bison are all - included. For UnixWare 7.1, this CD is now labeled "Open License - Software Supplement". If you do not have this CD, the software - on it is available - from . - - - - Skunkware has different versions for UnixWare and OpenServer. - Make sure you install the correct version for your operating - system, except as noted below. - - - - On UnixWare 7.1.3 and beyond, the GCC compiler is included on the - UDK CD as is GNU Make. - - - - - GNU Make - - - You need to use the GNU Make program, which is on the Skunkware - CD. By default, it installs - as /usr/local/bin/make. - - - - As of UnixWare 7.1.3 and above, the GNU Make program is the - OSTK portion of the UDK CD, and is - in /usr/gnu/bin/gmake. - - - - - Readline - - - The Readline library is on the Skunkware CD. But it is not - included on the UnixWare 7.1 Skunkware CD. If you have the - UnixWare 7.0.0 or 7.0.1 Skunkware CDs, you can install it from - there. Otherwise, - try . - - - - By default, Readline installs into /usr/local/lib and - /usr/local/include. However, the - PostgreSQL configure program will not find it - there without help. If you installed Readline, then use the - following options to configure: - -./configure --with-libraries=/usr/local/lib --with-includes=/usr/local/include - - - - - - Using the UDK on OpenServer - - - If you are using the new Universal Development Kit (UDK) compiler - on OpenServer, you need to specify the locations of the UDK - libraries: - -./configure --with-libraries=/udk/usr/lib --with-includes=/udk/usr/include - - Putting these together with the Readline options from above: - -./configure --with-libraries="/udk/usr/lib /usr/local/lib" --with-includes="/udk/usr/include /usr/local/include" - - - - - - Reading the PostgreSQL Man Pages - - - By default, the PostgreSQL man pages are installed into - /usr/local/pgsql/share/man. By default, UnixWare - does not look there for man pages. To be able to read them you - need to modify the - MANPATH variable - in /etc/default/man, for example: - -MANPATH=/usr/lib/scohelp/%L/man:/usr/dt/man:/usr/man:/usr/share/man:scohelp:/usr/local/man:/usr/local/pgsql/share/man - - - - - On OpenServer, some extra research needs to be invested to make - the man pages usable, because the man system is a bit different - from other platforms. Currently, PostgreSQL will not install - them at all. - - - - - C99 Issues with the 7.1.1b Feature Supplement - - - For compilers earlier than the one released with OpenUNIX 8.0.0 - (UnixWare 7.1.2), including the 7.1.1b Feature Supplement, you - may need to specify - in CFLAGS or the CC - environment variable. The indication of this is an error in - compiling tuplesort.c referencing inline - functions. Apparently there was a change in the 7.1.2(8.0.0) - compiler and beyond. - - - - - Threading on UnixWare - - - For threading, youmust use - on all libpq-using programs. libpq - uses pthread_* calls, which are only - available with the - - - - Solaris diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index a8efb3d006..6ae62b4d2b 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1086,34 +1086,6 @@ sysctl -w kern.sysv.shmall - - SCO OpenServer - SCO OpenServerIPC configuration - - - - In the default configuration, only 512 kB of shared memory per - segment is allowed. To increase the setting, first change to the - directory /etc/conf/cf.d. To display the current value of - SHMMAX, run: - -./configure -y SHMMAX - - To set a new value for SHMMAX, run: - -./configure SHMMAX=value - - where value is the new value you want to use - (in bytes). After setting SHMMAX, rebuild the kernel: - -./link_unix - - and reboot. - - - - - Solaris 2.6 to 2.9 (Solaris 6 to Solaris 9) @@ -1189,36 +1161,6 @@ project.max-msg-ids=(priv,4096,deny) - - - UnixWare - UnixWareIPC configuration - - - - On UnixWare 7, the maximum size for shared - memory segments is 512 kB in the default configuration. - To display the current value of SHMMAX, run: - -/etc/conf/bin/idtune -g SHMMAX - - which displays the current, default, minimum, and maximum - values. To set a new value for SHMMAX, - run: - -/etc/conf/bin/idtune SHMMAX value - - where value is the new value you want to use - (in bytes). After setting SHMMAX, rebuild the - kernel: - -/etc/conf/bin/idbuild -B - - and reboot. - - - - diff --git a/src/Makefile.global.in b/src/Makefile.global.in index e1e2c0adaf..ea61eb518f 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -265,7 +265,6 @@ UUID_LIBS = @UUID_LIBS@ UUID_EXTRA_OBJS = @UUID_EXTRA_OBJS@ LD = @LD@ with_gnu_ld = @with_gnu_ld@ -ld_R_works = @ld_R_works@ # We want -L for libpgport.a and libpgcommon.a to be first in LDFLAGS. We # also need LDFLAGS to be a "recursively expanded" variable, else adjustments diff --git a/src/Makefile.shlib b/src/Makefile.shlib index 87c80c5d01..358d90837c 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -236,30 +236,6 @@ ifeq ($(PORTNAME), solaris) endif endif -ifeq ($(PORTNAME), sco) - ifeq ($(GCC), yes) - LINK.shared = $(CC) -shared - else - LINK.shared = $(CC) -G - endif - LINK.shared += -Wl,-z,text - ifdef soname - LINK.shared += -Wl,-h,$(soname) - endif -endif - -ifeq ($(PORTNAME), unixware) - ifeq ($(GCC), yes) - LINK.shared = $(CC) -shared - else - LINK.shared = $(CC) -G - endif - LINK.shared += -Wl,-z,text - ifdef soname - LINK.shared += -Wl,-h,$(soname) - endif -endif - ifeq ($(PORTNAME), cygwin) LINK.shared = $(CC) -shared ifdef SO_MAJOR_VERSION diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index bae96bf18f..affa9bb7ab 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -683,16 +683,6 @@ StreamConnection(pgsocket server_fd, Port *port) return STATUS_ERROR; } -#ifdef SCO_ACCEPT_BUG - - /* - * UnixWare 7+ and OpenServer 5.0.4 are known to have this bug, but it - * shouldn't hurt to catch it for all versions of those platforms. - */ - if (port->raddr.addr.ss_family == 0) - port->raddr.addr.ss_family = AF_UNIX; -#endif - /* fill in the server (local) address */ port->laddr.salen = sizeof(port->laddr.addr); if (getsockname(port->sock, diff --git a/src/backend/port/dynloader/sco.c b/src/backend/port/dynloader/sco.c deleted file mode 100644 index 1e24f494ac..0000000000 --- a/src/backend/port/dynloader/sco.c +++ /dev/null @@ -1,7 +0,0 @@ -/* - * src/backend/port/dynloader/sco.c - * - * Dummy file used for nothing at this point - * - * see sco.h - */ diff --git a/src/backend/port/dynloader/sco.h b/src/backend/port/dynloader/sco.h deleted file mode 100644 index 86f2383729..0000000000 --- a/src/backend/port/dynloader/sco.h +++ /dev/null @@ -1,46 +0,0 @@ -/*------------------------------------------------------------------------- - * - * sco.h - * port-specific prototypes for SCO 3.2v5.2 - * - * - * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/backend/port/dynloader/sco.h - * - *------------------------------------------------------------------------- - */ -#ifndef PORT_PROTOS_H -#define PORT_PROTOS_H - -#include -#include "utils/dynamic_loader.h" /* pgrminclude ignore */ - -/* - * Dynamic Loader on SCO 3.2v5.0.2 - * - * this dynamic loader uses the system dynamic loading interface for shared - * libraries (ie. dlopen/dlsym/dlclose). The user must specify a shared - * library as the file to be dynamically loaded. - */ - -/* - * In some older systems, the RTLD_NOW flag isn't defined and the mode - * argument to dlopen must always be 1. The RTLD_GLOBAL flag is wanted - * if available, but it doesn't exist everywhere. - * If it doesn't exist, set it to 0 so it has no effect. - */ -#ifndef RTLD_NOW -#define RTLD_NOW 1 -#endif -#ifndef RTLD_GLOBAL -#define RTLD_GLOBAL 0 -#endif - -#define pg_dlopen(f) dlopen((f), RTLD_NOW | RTLD_GLOBAL) -#define pg_dlsym dlsym -#define pg_dlclose dlclose -#define pg_dlerror dlerror - -#endif /* PORT_PROTOS_H */ diff --git a/src/backend/port/dynloader/unixware.c b/src/backend/port/dynloader/unixware.c deleted file mode 100644 index afb36dfe99..0000000000 --- a/src/backend/port/dynloader/unixware.c +++ /dev/null @@ -1,7 +0,0 @@ -/* - * src/backend/port/dynloader/unixware.c - * - * Dummy file used for nothing at this point - * - * see unixware.h - */ diff --git a/src/backend/port/dynloader/unixware.h b/src/backend/port/dynloader/unixware.h deleted file mode 100644 index 130a9a25d5..0000000000 --- a/src/backend/port/dynloader/unixware.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * src/backend/port/dynloader/unixware.h - * - *------------------------------------------------------------------------- - * - * unixware.h - * port-specific prototypes for Intel x86/UNIXWARE 7 - * - * - * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * unixware.h,v 1.2 1995/03/17 06:40:18 andrew Exp - * - *------------------------------------------------------------------------- - */ -#ifndef PORT_PROTOS_H -#define PORT_PROTOS_H - -#include -#include "utils/dynamic_loader.h" /* pgrminclude ignore */ - -/* - * Dynamic Loader on UnixWare. - * - * this dynamic loader uses the system dynamic loading interface for shared - * libraries (ie. dlopen/dlsym/dlclose). The user must specify a shared - * library as the file to be dynamically loaded. - */ - -/* - * In some older systems, the RTLD_NOW flag isn't defined and the mode - * argument to dlopen must always be 1. The RTLD_GLOBAL flag is wanted - * if available, but it doesn't exist everywhere. - * If it doesn't exist, set it to 0 so it has no effect. - */ -#ifndef RTLD_NOW -#define RTLD_NOW 1 -#endif -#ifndef RTLD_GLOBAL -#define RTLD_GLOBAL 0 -#endif - -#define pg_dlopen(f) dlopen((f), RTLD_NOW | RTLD_GLOBAL) -#define pg_dlsym dlsym -#define pg_dlclose dlclose -#define pg_dlerror dlerror - -#endif /* PORT_PROTOS_H */ diff --git a/src/include/port/sco.h b/src/include/port/sco.h deleted file mode 100644 index 30811450c9..0000000000 --- a/src/include/port/sco.h +++ /dev/null @@ -1,7 +0,0 @@ -/* - * src/include/port/sco.h - * - * see src/backend/libpq/pqcomm.c */ -#define SCO_ACCEPT_BUG - -#define USE_UNIVEL_CC diff --git a/src/include/port/unixware.h b/src/include/port/unixware.h deleted file mode 100644 index e068820957..0000000000 --- a/src/include/port/unixware.h +++ /dev/null @@ -1,11 +0,0 @@ -/* - * src/include/port/unixware.h - * - * see src/backend/libpq/pqcomm.c */ -#define SCO_ACCEPT_BUG - -/*************************************** - * Define this if you are compiling with - * the native UNIXWARE C compiler. - ***************************************/ -#define USE_UNIVEL_CC diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h index 7aad2de43d..3fe29cede6 100644 --- a/src/include/storage/s_lock.h +++ b/src/include/storage/s_lock.h @@ -706,29 +706,6 @@ typedef unsigned char slock_t; #if !defined(HAS_TEST_AND_SET) /* We didn't trigger above, let's try here */ -#if defined(USE_UNIVEL_CC) /* Unixware compiler */ -#define HAS_TEST_AND_SET - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -asm int -tas(volatile slock_t *s_lock) -{ -/* UNIVEL wants %mem in column 1, so we don't pgindent this file */ -%mem s_lock - pushl %ebx - movl s_lock, %ebx - movl $255, %eax - lock - xchgb %al, (%ebx) - popl %ebx -} - -#endif /* defined(USE_UNIVEL_CC) */ - - #if defined(__hppa) || defined(__hppa__) /* HP PA-RISC, GCC and HP compilers */ /* * HP's PA-RISC diff --git a/src/makefiles/Makefile.sco b/src/makefiles/Makefile.sco deleted file mode 100644 index 993861570a..0000000000 --- a/src/makefiles/Makefile.sco +++ /dev/null @@ -1,13 +0,0 @@ -AROPT = cr -export_dynamic = -Wl,-Bexport - -DLSUFFIX = .so -ifeq ($(GCC), yes) -CFLAGS_SL = -fpic -else -CFLAGS_SL = -K PIC -endif - -# Rule for building a shared library from a single .o file -%.so: %.o - $(LD) -G -Bdynamic -o $@ $< diff --git a/src/makefiles/Makefile.unixware b/src/makefiles/Makefile.unixware deleted file mode 100644 index a52717b268..0000000000 --- a/src/makefiles/Makefile.unixware +++ /dev/null @@ -1,35 +0,0 @@ -AROPT = crs -ifeq ($(with_gnu_ld), yes) - export_dynamic = -Wl,-E -else - export_dynamic = -Wl,-Bexport -endif - -ifeq ($(ld_R_works), yes) -ifeq ($(with_gnu_ld), yes) - rpath = -Wl,-rpath,'$(rpathdir)' -else - rpath = -Wl,-R'$(rpathdir)' -endif -endif - -# Unixware needs threads for everything that uses libpq -CFLAGS += $(PTHREAD_CFLAGS) - -DLSUFFIX = .so -ifeq ($(GCC), yes) -CFLAGS_SL = -fpic -else -CFLAGS_SL = -K PIC -endif -ifeq ($(GCC), yes) -SO_FLAGS = -shared -else -SO_FLAGS = -G -endif - -# Rule for building a shared library from a single .o file -%.so: %.o - $(CC) $(CFLAGS) $(LDFLAGS) $(LDFLAGS_SL) $(SO_FLAGS) -o $@ $< - -sqlmansect = 5sql diff --git a/src/port/getrusage.c b/src/port/getrusage.c index a6f1ef2681..d24af92339 100644 --- a/src/port/getrusage.c +++ b/src/port/getrusage.c @@ -18,7 +18,6 @@ #include "rusagestub.h" /* This code works on: - * sco * solaris_i386 * solaris_sparc * hpux 9.* diff --git a/src/template/sco b/src/template/sco deleted file mode 100644 index 9a736da8be..0000000000 --- a/src/template/sco +++ /dev/null @@ -1 +0,0 @@ -CC="$CC -b elf" diff --git a/src/template/unixware b/src/template/unixware deleted file mode 100644 index d08fca1e6b..0000000000 --- a/src/template/unixware +++ /dev/null @@ -1,41 +0,0 @@ -if test "$GCC" != yes; then - # The -Kno_host is for a bug in the compiler. See -hackers - # discussion on 7-8/Aug/2003. - cat >conftest.c <<__EOF__ -extern char *strcpy(char *, const char *); - -static void f(char *p, int n){ - strcpy(p+n,""); -} -void g(void){ - f(0, 0); -} -__EOF__ - - # Debugging and optimization are mutually exclusive - if test "$enable_debug" != yes; then - CFLAGS="-O" - fi - if $CC -c -O -Kinline conftest.c >conftest.err 2>&1; then - CFLAGS="$CFLAGS -Kinline" - else - CFLAGS="$CFLAGS -Kinline,no_host" - fi - rm -f conftest.* - - PTHREAD_CFLAGS="-Kpthread" - -# The effect of doing threading for the backend does not work -# because of a threading bug that appears in the regression tests: -# -# in make check, the plpgsql test (plpgsql.sql) -# set statement_timeout to 1000; -# select blockme(); -# reset statement_timeout; -# -# per report from Olivier PRENANT - -fi - -# Unixware's ldap library reportedly needs these too -EXTRA_LDAP_LIBS="-llber -lresolv" From 2f1eaf87e868a1c42f2b159958623daa6a666de4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 11 Oct 2016 12:19:18 -0400 Subject: [PATCH 307/871] Drop server support for FE/BE protocol version 1.0. While this isn't a lot of code, it's been essentially untestable for a very long time, because libpq doesn't support anything older than protocol 2.0, and has not since release 6.3. There's no reason to believe any other client-side code still uses that protocol, either. Discussion: <2661.1475849167@sss.pgh.pa.us> --- src/backend/access/common/printtup.c | 4 +--- src/backend/commands/copy.c | 28 ++-------------------------- src/backend/tcop/dest.c | 6 +++--- src/backend/tcop/postgres.c | 3 +-- src/include/libpq/pqcomm.h | 2 +- 5 files changed, 8 insertions(+), 35 deletions(-) diff --git a/src/backend/access/common/printtup.c b/src/backend/access/common/printtup.c index d213af9074..e2b1050c96 100644 --- a/src/backend/access/common/printtup.c +++ b/src/backend/access/common/printtup.c @@ -229,9 +229,7 @@ SendRowDescriptionMessage(TupleDesc typeinfo, List *targetlist, int16 *formats) atttypid = getBaseTypeAndTypmod(atttypid, &atttypmod); pq_sendint(&buf, (int) atttypid, sizeof(atttypid)); pq_sendint(&buf, attrs[i]->attlen, sizeof(attrs[i]->attlen)); - /* typmod appears in protocol 2.0 and up */ - if (proto >= 2) - pq_sendint(&buf, atttypmod, sizeof(atttypmod)); + pq_sendint(&buf, atttypmod, sizeof(atttypmod)); /* format info appears in protocol 3.0 and up */ if (proto >= 3) { diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 457c9bbd74..b4140eb68a 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -353,7 +353,7 @@ SendCopyBegin(CopyState cstate) pq_endmessage(&buf); cstate->copy_dest = COPY_NEW_FE; } - else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2) + else { /* old way */ if (cstate->binary) @@ -365,18 +365,6 @@ SendCopyBegin(CopyState cstate) pq_startcopyout(); cstate->copy_dest = COPY_OLD_FE; } - else - { - /* very old way */ - if (cstate->binary) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("COPY BINARY is not supported to stdout or from stdin"))); - pq_putemptymessage('B'); - /* grottiness needed for old COPY OUT protocol */ - pq_startcopyout(); - cstate->copy_dest = COPY_OLD_FE; - } } static void @@ -399,7 +387,7 @@ ReceiveCopyBegin(CopyState cstate) cstate->copy_dest = COPY_NEW_FE; cstate->fe_msgbuf = makeStringInfo(); } - else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2) + else { /* old way */ if (cstate->binary) @@ -411,18 +399,6 @@ ReceiveCopyBegin(CopyState cstate) pq_startmsgread(); cstate->copy_dest = COPY_OLD_FE; } - else - { - /* very old way */ - if (cstate->binary) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("COPY BINARY is not supported to stdout or from stdin"))); - pq_putemptymessage('D'); - /* any error in old protocol will make us lose sync */ - pq_startmsgread(); - cstate->copy_dest = COPY_OLD_FE; - } /* We *must* flush here to ensure FE knows it can send. */ pq_flush(); } diff --git a/src/backend/tcop/dest.c b/src/backend/tcop/dest.c index de45cbc4fb..60a92801f1 100644 --- a/src/backend/tcop/dest.c +++ b/src/backend/tcop/dest.c @@ -218,8 +218,8 @@ NullCommand(CommandDest dest) /* ---------------- * ReadyForQuery - tell dest that we are ready for a new query * - * The ReadyForQuery message is sent in protocol versions 2.0 and up - * so that the FE can tell when we are done processing a query string. + * The ReadyForQuery message is sent so that the FE can tell when + * we are done processing a query string. * In versions 3.0 and up, it also carries a transaction state indicator. * * Note that by flushing the stdio buffer here, we can avoid doing it @@ -241,7 +241,7 @@ ReadyForQuery(CommandDest dest) pq_sendbyte(&buf, TransactionBlockStatusCode()); pq_endmessage(&buf); } - else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2) + else pq_putemptymessage('Z'); /* Flush output at end of cycle in any case. */ pq_flush(); diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 98ccbbb4d1..2347f1bcdc 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3768,8 +3768,7 @@ PostgresMain(int argc, char *argv[], /* * Send this backend's cancellation info to the frontend. */ - if (whereToSendOutput == DestRemote && - PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2) + if (whereToSendOutput == DestRemote) { StringInfoData buf; diff --git a/src/include/libpq/pqcomm.h b/src/include/libpq/pqcomm.h index c6bbfc2377..96484227f7 100644 --- a/src/include/libpq/pqcomm.h +++ b/src/include/libpq/pqcomm.h @@ -107,7 +107,7 @@ typedef struct /* The earliest and latest frontend/backend protocol version supported. */ -#define PG_PROTOCOL_EARLIEST PG_PROTOCOL(1,0) +#define PG_PROTOCOL_EARLIEST PG_PROTOCOL(2,0) #define PG_PROTOCOL_LATEST PG_PROTOCOL(3,0) typedef uint32 ProtocolVersion; /* FE/BE protocol version number */ From b75f467b6eec0678452fd8d7f8d306e6df3a1076 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 12 Oct 2016 12:05:45 +0300 Subject: [PATCH 308/871] Simplify the code for logical tape read buffers. Pass the buffer size as argument to LogicalTapeRewindForRead, rather than setting it earlier with the separate LogicTapeAssignReadBufferSize call. This way, the buffer size is set closer to where it's actually used, which makes the code easier to understand. This makes the calculation for how much memory to use for the buffers less precise. We now use the same amount of memory for every tape, rounded down to the nearest BLCKSZ boundary, instead of using one more block for some tapes, to get the total up to exact amount of memory available. That should be OK, merging isn't too sensitive to the exact amount of memory used. Reviewed by Peter Geoghegan Discussion: <0f607c4b-df23-353e-bf56-c0389d28495f@iki.fi> --- src/backend/utils/sort/logtape.c | 213 ++++++++++++++--------------- src/backend/utils/sort/tuplesort.c | 120 +++++----------- src/include/utils/logtape.h | 6 +- 3 files changed, 139 insertions(+), 200 deletions(-) diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index 6cc06b24e0..316301061d 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -141,14 +141,6 @@ typedef struct LogicalTape long curBlockNumber; /* this block's logical blk# within tape */ int pos; /* next read/write position in buffer */ int nbytes; /* total # of valid bytes in buffer */ - - /* - * Desired buffer size to use when reading. To keep things simple, we use - * a single-block buffer when writing, or when reading a frozen tape. But - * when we are reading and will only read forwards, we allocate a larger - * buffer, determined by read_buffer_size. - */ - int read_buffer_size; } LogicalTape; /* @@ -609,7 +601,6 @@ LogicalTapeSetCreate(int ntapes) lt->lastBlockBytes = 0; lt->buffer = NULL; lt->buffer_size = 0; - lt->read_buffer_size = BLCKSZ; lt->curBlockNumber = 0L; lt->pos = 0; lt->nbytes = 0; @@ -739,13 +730,20 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum, } /* - * Rewind logical tape and switch from writing to reading or vice versa. + * Rewind logical tape and switch from writing to reading. * - * Unless the tape has been "frozen" in read state, forWrite must be the - * opposite of the previous tape state. + * The tape must currently be in writing state, or "frozen" in read state. + * + * 'buffer_size' specifies how much memory to use for the read buffer. It + * does not include the memory needed for the indirect blocks. Regardless + * of the argument, the actual amount of memory used is between BLCKSZ and + * MaxAllocSize, and is a multiple of BLCKSZ. The given value is rounded + * down and truncated to fit those constraints, if necessary. If the tape + * is frozen, the 'buffer_size' argument is ignored, and a small BLCKSZ byte + * buffer is used. */ void -LogicalTapeRewind(LogicalTapeSet *lts, int tapenum, bool forWrite) +LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size) { LogicalTape *lt; long datablocknum; @@ -753,88 +751,112 @@ LogicalTapeRewind(LogicalTapeSet *lts, int tapenum, bool forWrite) Assert(tapenum >= 0 && tapenum < lts->nTapes); lt = <s->tapes[tapenum]; - if (!forWrite) + /* + * Round and cap buffer_size if needed. + */ + if (lt->frozen) + buffer_size = BLCKSZ; + else { - if (lt->writing) - { - /* - * Completion of a write phase. Flush last partial data block, - * flush any partial indirect blocks, rewind for normal - * (destructive) read. - */ - if (lt->dirty) - ltsDumpBuffer(lts, lt); - lt->lastBlockBytes = lt->nbytes; - lt->writing = false; - datablocknum = ltsRewindIndirectBlock(lts, lt->indirect, false); - } - else - { - /* - * This is only OK if tape is frozen; we rewind for (another) read - * pass. - */ - Assert(lt->frozen); - datablocknum = ltsRewindFrozenIndirectBlock(lts, lt->indirect); - } + /* need at least one block */ + if (buffer_size < BLCKSZ) + buffer_size = BLCKSZ; - /* Allocate a read buffer (unless the tape is empty) */ - if (lt->buffer) - pfree(lt->buffer); - lt->buffer = NULL; - lt->buffer_size = 0; - if (datablocknum != -1L) - { - lt->buffer = palloc(lt->read_buffer_size); - lt->buffer_size = lt->read_buffer_size; - } + /* + * palloc() larger than MaxAllocSize would fail (a multi-gigabyte + * buffer is unlikely to be helpful, anyway) + */ + if (buffer_size > MaxAllocSize) + buffer_size = MaxAllocSize; - /* Read the first block, or reset if tape is empty */ - lt->curBlockNumber = 0L; - lt->pos = 0; - lt->nbytes = 0; - if (datablocknum != -1L) - ltsReadFillBuffer(lts, lt, datablocknum); + /* round down to BLCKSZ boundary */ + buffer_size -= buffer_size % BLCKSZ; + } + + if (lt->writing) + { + /* + * Completion of a write phase. Flush last partial data block, flush + * any partial indirect blocks, rewind for normal (destructive) read. + */ + if (lt->dirty) + ltsDumpBuffer(lts, lt); + lt->lastBlockBytes = lt->nbytes; + lt->writing = false; + datablocknum = ltsRewindIndirectBlock(lts, lt->indirect, false); } else { /* - * Completion of a read phase. Rewind and prepare for write. - * - * NOTE: we assume the caller has read the tape to the end; otherwise - * untouched data and indirect blocks will not have been freed. We - * could add more code to free any unread blocks, but in current usage - * of this module it'd be useless code. + * This is only OK if tape is frozen; we rewind for (another) read + * pass. */ - IndirectBlock *ib, - *nextib; + Assert(lt->frozen); + datablocknum = ltsRewindFrozenIndirectBlock(lts, lt->indirect); + } - Assert(!lt->writing && !lt->frozen); - /* Must truncate the indirect-block hierarchy down to one level. */ - if (lt->indirect) - { - for (ib = lt->indirect->nextup; ib != NULL; ib = nextib) - { - nextib = ib->nextup; - pfree(ib); - } - lt->indirect->nextSlot = 0; - lt->indirect->nextup = NULL; - } - lt->writing = true; - lt->dirty = false; - lt->numFullBlocks = 0L; - lt->lastBlockBytes = 0; - lt->curBlockNumber = 0L; - lt->pos = 0; - lt->nbytes = 0; + /* Allocate a read buffer (unless the tape is empty) */ + if (lt->buffer) + pfree(lt->buffer); + lt->buffer = NULL; + lt->buffer_size = 0; + if (datablocknum != -1L) + { + lt->buffer = palloc(buffer_size); + lt->buffer_size = buffer_size; + } - if (lt->buffer) + /* Read the first block, or reset if tape is empty */ + lt->curBlockNumber = 0L; + lt->pos = 0; + lt->nbytes = 0; + if (datablocknum != -1L) + ltsReadFillBuffer(lts, lt, datablocknum); +} + +/* + * Rewind logical tape and switch from reading to writing. + * + * NOTE: we assume the caller has read the tape to the end; otherwise + * untouched data and indirect blocks will not have been freed. We + * could add more code to free any unread blocks, but in current usage + * of this module it'd be useless code. + */ +void +LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum) +{ + LogicalTape *lt; + IndirectBlock *ib, + *nextib; + + Assert(tapenum >= 0 && tapenum < lts->nTapes); + lt = <s->tapes[tapenum]; + + Assert(!lt->writing && !lt->frozen); + /* Must truncate the indirect-block hierarchy down to one level. */ + if (lt->indirect) + { + for (ib = lt->indirect->nextup; ib != NULL; ib = nextib) { - pfree(lt->buffer); - lt->buffer = NULL; - lt->buffer_size = 0; + nextib = ib->nextup; + pfree(ib); } + lt->indirect->nextSlot = 0; + lt->indirect->nextup = NULL; + } + lt->writing = true; + lt->dirty = false; + lt->numFullBlocks = 0L; + lt->lastBlockBytes = 0; + lt->curBlockNumber = 0L; + lt->pos = 0; + lt->nbytes = 0; + + if (lt->buffer) + { + pfree(lt->buffer); + lt->buffer = NULL; + lt->buffer_size = 0; } } @@ -1105,28 +1127,3 @@ LogicalTapeSetBlocks(LogicalTapeSet *lts) { return lts->nFileBlocks; } - -/* - * Set buffer size to use, when reading from given tape. - */ -void -LogicalTapeAssignReadBufferSize(LogicalTapeSet *lts, int tapenum, size_t avail_mem) -{ - LogicalTape *lt; - - Assert(tapenum >= 0 && tapenum < lts->nTapes); - lt = <s->tapes[tapenum]; - - /* - * The buffer size must be a multiple of BLCKSZ in size, so round the - * given value down to nearest BLCKSZ. Make sure we have at least one - * page. Also, don't go above MaxAllocSize, to avoid erroring out. A - * multi-gigabyte buffer is unlikely to be helpful, anyway. - */ - if (avail_mem < BLCKSZ) - avail_mem = BLCKSZ; - if (avail_mem > MaxAllocSize) - avail_mem = MaxAllocSize; - avail_mem -= avail_mem % BLCKSZ; - lt->read_buffer_size = avail_mem; -} diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index dd83e7a8f2..f6eb30c2ce 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -366,6 +366,9 @@ struct Tuplesortstate char *slabMemoryEnd; /* end of slab memory arena */ SlabSlot *slabFreeHead; /* head of free list */ + /* Buffer size to use for reading input tapes, during merge. */ + size_t read_buffer_size; + /* * When we return a tuple to the caller in tuplesort_gettuple_XXX, that * came from a tape (that is, in TSS_SORTEDONTAPE or TSS_FINALMERGE @@ -579,7 +582,6 @@ static bool useselection(Tuplesortstate *state); static void inittapes(Tuplesortstate *state); static void selectnewtape(Tuplesortstate *state); static void init_slab_allocator(Tuplesortstate *state, int numSlots); -static void init_tape_buffers(Tuplesortstate *state, int numInputTapes); static void mergeruns(Tuplesortstate *state); static void mergeonerun(Tuplesortstate *state); static void beginmerge(Tuplesortstate *state); @@ -2056,7 +2058,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward, * end of the sort anyway, but better to release the * memory early. */ - LogicalTapeRewind(state->tapeset, srcTape, true); + LogicalTapeRewindForWrite(state->tapeset, srcTape); return true; } newtup.tupindex = srcTape; @@ -2511,72 +2513,6 @@ init_slab_allocator(Tuplesortstate *state, int numSlots) state->slabAllocatorUsed = true; } -/* - * Divide all remaining work memory (availMem) as read buffers, for all - * the tapes that will be used during the merge. - * - * We use the number of possible *input* tapes here, rather than maxTapes, - * for the calculation. At all times, we'll be reading from at most - * numInputTapes tapes, and one tape is used for output (unless we do an - * on-the-fly final merge, in which case we don't have an output tape). - */ -static void -init_tape_buffers(Tuplesortstate *state, int numInputTapes) -{ - int64 availBlocks; - int64 blocksPerTape; - int remainder; - int tapenum; - - /* - * Divide availMem evenly among the number of input tapes. - */ - availBlocks = state->availMem / BLCKSZ; - blocksPerTape = availBlocks / numInputTapes; - remainder = availBlocks % numInputTapes; - USEMEM(state, availBlocks * BLCKSZ); - -#ifdef TRACE_SORT - if (trace_sort) - elog(LOG, "using " INT64_FORMAT " KB of memory for read buffers among %d input tapes", - (availBlocks * BLCKSZ) / 1024, numInputTapes); -#endif - - /* - * Use one page per tape, even if we are out of memory. - * tuplesort_merge_order() should've chosen the number of tapes so that - * this can't happen, but better safe than sorry. (This also protects - * from a negative availMem.) - */ - if (blocksPerTape < 1) - { - blocksPerTape = 1; - remainder = 0; - } - - /* - * Set the buffers for the tapes. - * - * In a multi-phase merge, the tape that is initially used as an output - * tape, will later be rewound and read from, and should also use a large - * buffer at that point. So we must loop up to maxTapes, not just - * numInputTapes! - * - * If there are fewer runs than tapes, we will set the buffer size also - * for tapes that will go completely unused, but that's harmless. - * LogicalTapeAssignReadBufferSize() doesn't allocate the buffer - * immediately, it just sets the size that will be used, when the tape is - * rewound for read, and the tape isn't empty. - */ - for (tapenum = 0; tapenum < state->maxTapes; tapenum++) - { - int64 numBlocks = blocksPerTape + (tapenum < remainder ? 1 : 0); - - LogicalTapeAssignReadBufferSize(state->tapeset, tapenum, - numBlocks * BLCKSZ); - } -} - /* * mergeruns -- merge all the completed initial runs. * @@ -2679,25 +2615,32 @@ mergeruns(Tuplesortstate *state) } /* - * Use all the spare memory we have available for read buffers for the - * tapes. + * Use all the spare memory we have available for read buffers among the + * input tapes. * * We do this only after checking for the case that we produced only one * initial run, because there is no need to use a large read buffer when * we're reading from a single tape. With one tape, the I/O pattern will * be the same regardless of the buffer size. * - * We don't try to "rebalance" the amount of memory among tapes, when we - * start a new merge phase, even if some tapes can be inactive in the - * phase. That would be hard, because logtape.c doesn't know where one - * run ends and another begins. When a new merge phase begins, and a tape - * doesn't participate in it, its buffer nevertheless already contains - * tuples from the next run on same tape, so we cannot release the buffer. - * That's OK in practice, merge performance isn't that sensitive to the - * amount of buffers used, and most merge phases use all or almost all - * tapes, anyway. + * We don't try to "rebalance" the memory among tapes, when we start a new + * merge phase, even if some tapes are inactive in the new phase. That + * would be hard, because logtape.c doesn't know where one run ends and + * another begins. When a new merge phase begins, and a tape doesn't + * participate in it, its buffer nevertheless already contains tuples from + * the next run on same tape, so we cannot release the buffer. That's OK + * in practice, merge performance isn't that sensitive to the amount of + * buffers used, and most merge phases use all or almost all tapes, + * anyway. */ - init_tape_buffers(state, numInputTapes); +#ifdef TRACE_SORT + if (trace_sort) + elog(LOG, "using " INT64_FORMAT " KB of memory for read buffers among %d input tapes", + (state->availMem) / 1024, numInputTapes); +#endif + + state->read_buffer_size = state->availMem / numInputTapes; + USEMEM(state, state->availMem); /* * Allocate a new 'memtuples' array, for the heap. It will hold one tuple @@ -2709,7 +2652,7 @@ mergeruns(Tuplesortstate *state) /* End of step D2: rewind all output tapes to prepare for merging */ for (tapenum = 0; tapenum < state->tapeRange; tapenum++) - LogicalTapeRewind(state->tapeset, tapenum, false); + LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size); for (;;) { @@ -2772,11 +2715,10 @@ mergeruns(Tuplesortstate *state) if (--state->Level == 0) break; /* rewind output tape T to use as new input */ - LogicalTapeRewind(state->tapeset, state->tp_tapenum[state->tapeRange], - false); + LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange], + state->read_buffer_size); /* rewind used-up input tape P, and prepare it for write pass */ - LogicalTapeRewind(state->tapeset, state->tp_tapenum[state->tapeRange - 1], - true); + LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]); state->tp_runs[state->tapeRange - 1] = 0; /* @@ -2812,7 +2754,7 @@ mergeruns(Tuplesortstate *state) for (tapenum = 0; tapenum < state->maxTapes; tapenum++) { if (tapenum != state->result_tape) - LogicalTapeRewind(state->tapeset, tapenum, true); + LogicalTapeRewindForWrite(state->tapeset, tapenum); } } @@ -3174,9 +3116,9 @@ tuplesort_rescan(Tuplesortstate *state) state->markpos_eof = false; break; case TSS_SORTEDONTAPE: - LogicalTapeRewind(state->tapeset, - state->result_tape, - false); + LogicalTapeRewindForRead(state->tapeset, + state->result_tape, + 0); state->eof_reached = false; state->markpos_block = 0L; state->markpos_offset = 0; diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h index 362a6196dc..d7dccb85be 100644 --- a/src/include/utils/logtape.h +++ b/src/include/utils/logtape.h @@ -31,7 +31,9 @@ extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum, void *ptr, size_t size); extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum, void *ptr, size_t size); -extern void LogicalTapeRewind(LogicalTapeSet *lts, int tapenum, bool forWrite); +extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, + size_t buffer_size); +extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum); extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum); extern bool LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size); @@ -39,8 +41,6 @@ extern bool LogicalTapeSeek(LogicalTapeSet *lts, int tapenum, long blocknum, int offset); extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum, long *blocknum, int *offset); -extern void LogicalTapeAssignReadBufferSize(LogicalTapeSet *lts, int tapenum, - size_t bufsize); extern long LogicalTapeSetBlocks(LogicalTapeSet *lts); #endif /* LOGTAPE_H */ From bb55dd6059dddf2cd44423da6a3f6946826953cf Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 12 Oct 2016 12:07:54 +0300 Subject: [PATCH 309/871] Fix copy-pasto in comment. Amit Langote --- src/include/catalog/pg_cast.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/catalog/pg_cast.h b/src/include/catalog/pg_cast.h index ee568d836a..04d11c0cfe 100644 --- a/src/include/catalog/pg_cast.h +++ b/src/include/catalog/pg_cast.h @@ -59,7 +59,7 @@ typedef enum CoercionCodes /* * The allowable values for pg_cast.castmethod are specified by this enum. - * Since castcontext is stored as a "char", we use ASCII codes for human + * Since castmethod is stored as a "char", we use ASCII codes for human * convenience in reading the table. */ typedef enum CoercionMethod From 64f3524e2c8deebc02808aa5ebdfa17859473add Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 12 Oct 2016 12:19:56 -0400 Subject: [PATCH 310/871] Remove pg_dump/pg_dumpall support for dumping from pre-8.0 servers. The need for dumping from such ancient servers has decreased to about nil in the field, so let's remove all the code that catered to it. Aside from removing a lot of boilerplate variant queries, this allows us to not have to cope with servers that don't have (a) schemas or (b) pg_depend. That means we can get rid of assorted squishy code around that. There may be some nonobvious additional simplifications possible, but this patch already removes about 1500 lines of code. I did not remove the ability for pg_restore to read custom-format archives generated by these old versions (and light testing says that that does still work). If you have an old server, you probably also have a pg_dump that will work with it; but you have an old custom-format backup file, that might be all you have. It'd be possible at this point to remove fmtQualifiedId()'s version argument, but I refrained since that would affect code outside pg_dump. Discussion: <2661.1475849167@sss.pgh.pa.us> --- doc/src/sgml/ref/pg_dump.sgml | 7 +- src/bin/pg_dump/dumputils.c | 50 +- src/bin/pg_dump/pg_backup_archiver.c | 2 +- src/bin/pg_dump/pg_dump.c | 1761 +++----------------------- src/bin/pg_dump/pg_dump.h | 1 - src/bin/pg_dump/pg_dump_sort.c | 96 +- src/bin/pg_dump/pg_dumpall.c | 110 +- 7 files changed, 250 insertions(+), 1777 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index be1b684082..371a61427d 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -758,10 +758,9 @@ PostgreSQL documentation the dump. Instead fail if unable to lock a table within the specified timeout. The timeout may be specified in any of the formats accepted by SET - statement_timeout. (Allowed values vary depending on the server + statement_timeout. (Allowed formats vary depending on the server version you are dumping from, but an integer number of milliseconds - is accepted by all versions since 7.3. This option is ignored when - dumping from a pre-7.3 server.) + is accepted by all versions.) @@ -1172,7 +1171,7 @@ CREATE DATABASE foo WITH TEMPLATE template0; PostgreSQL server versions newer than pg_dump's version. pg_dump can also dump from PostgreSQL servers older than its own version. - (Currently, servers back to version 7.0 are supported.) + (Currently, servers back to version 8.0 are supported.) However, pg_dump cannot dump from PostgreSQL servers newer than its own major version; it will refuse to even try, rather than risk making an invalid dump. diff --git a/src/bin/pg_dump/dumputils.c b/src/bin/pg_dump/dumputils.c index cd1e8c4a68..0d5166891e 100644 --- a/src/bin/pg_dump/dumputils.c +++ b/src/bin/pg_dump/dumputils.c @@ -18,8 +18,6 @@ #include "fe_utils/string_utils.h" -#define supports_grant_options(version) ((version) >= 70400) - static bool parseAclItem(const char *item, const char *type, const char *name, const char *subname, int remoteVersion, PQExpBuffer grantee, PQExpBuffer grantor, @@ -246,11 +244,9 @@ buildACLCommands(const char *name, const char *subname, /* * For the owner, the default privilege level is ALL WITH - * GRANT OPTION (only ALL prior to 7.4). + * GRANT OPTION. */ - if (supports_grant_options(remoteVersion) - ? strcmp(privswgo->data, "ALL") != 0 - : strcmp(privs->data, "ALL") != 0) + if (strcmp(privswgo->data, "ALL") != 0) { appendPQExpBuffer(firstsql, "%sREVOKE ALL", prefix); if (subname) @@ -403,16 +399,19 @@ buildDefaultACLCommands(const char *type, const char *nspname, * username=privilegecodes/grantor * or * group groupname=privilegecodes/grantor - * (the /grantor part will not be present if pre-7.4 database). + * (the "group" case occurs only with servers before 8.1). + * + * Returns true on success, false on parse error. On success, the components + * of the string are returned in the PQExpBuffer parameters. * * The returned grantee string will be the dequoted username or groupname - * (preceded with "group " in the latter case). The returned grantor is - * the dequoted grantor name or empty. Privilege characters are decoded - * and split between privileges with grant option (privswgo) and without - * (privs). + * (preceded with "group " in the latter case). Note that a grant to PUBLIC + * is represented by an empty grantee string. The returned grantor is the + * dequoted grantor name. Privilege characters are decoded and split between + * privileges with grant option (privswgo) and without (privs). * - * Note: for cross-version compatibility, it's important to use ALL when - * appropriate. + * Note: for cross-version compatibility, it's important to use ALL to + * represent the privilege sets whenever appropriate. */ static bool parseAclItem(const char *item, const char *type, @@ -439,7 +438,7 @@ parseAclItem(const char *item, const char *type, return false; } - /* grantor may be listed after / */ + /* grantor should appear after / */ slpos = strchr(eqpos + 1, '/'); if (slpos) { @@ -452,7 +451,10 @@ parseAclItem(const char *item, const char *type, } } else - resetPQExpBuffer(grantor); + { + free(buf); + return false; + } /* privilege codes */ #define CONVERT_PRIV(code, keywd) \ @@ -490,29 +492,19 @@ do { \ { /* table only */ CONVERT_PRIV('a', "INSERT"); - if (remoteVersion >= 70200) - CONVERT_PRIV('x', "REFERENCES"); + CONVERT_PRIV('x', "REFERENCES"); /* rest are not applicable to columns */ if (subname == NULL) { - if (remoteVersion >= 70200) - { - CONVERT_PRIV('d', "DELETE"); - CONVERT_PRIV('t', "TRIGGER"); - } + CONVERT_PRIV('d', "DELETE"); + CONVERT_PRIV('t', "TRIGGER"); if (remoteVersion >= 80400) CONVERT_PRIV('D', "TRUNCATE"); } } /* UPDATE */ - if (remoteVersion >= 70200 || - strcmp(type, "SEQUENCE") == 0 || - strcmp(type, "SEQUENCES") == 0) - CONVERT_PRIV('w', "UPDATE"); - else - /* 7.0 and 7.1 have a simpler worldview */ - CONVERT_PRIV('w', "UPDATE,DELETE"); + CONVERT_PRIV('w', "UPDATE"); } else if (strcmp(type, "FUNCTION") == 0 || strcmp(type, "FUNCTIONS") == 0) diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index bba8b6ca9f..e237b4a9c9 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -388,7 +388,7 @@ RestoreArchive(Archive *AHX) * target. */ AHX->minRemoteVersion = 0; - AHX->maxRemoteVersion = 999999; + AHX->maxRemoteVersion = 9999999; ConnectDatabase(AHX, ropt->dbname, ropt->pghost, ropt->pgport, ropt->username, diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index fde7f59c3d..c821f3b0ee 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -96,9 +96,6 @@ bool g_verbose; /* User wants verbose narration of our /* subquery used to convert user ID (eg, datdba) to user name */ static const char *username_subquery; -/* obsolete as of 7.3: */ -static Oid g_last_builtin_oid; /* value of the last builtin oid */ - /* The specified names/patterns should to match at least one entity */ static int strict_names = 0; @@ -142,7 +139,7 @@ static void expand_table_name_patterns(Archive *fout, SimpleStringList *patterns, SimpleOidList *oids, bool strict_names); -static NamespaceInfo *findNamespace(Archive *fout, Oid nsoid, Oid objoid); +static NamespaceInfo *findNamespace(Archive *fout, Oid nsoid); static void dumpTableData(Archive *fout, TableDataInfo *tdinfo); static void refreshMatViewData(Archive *fout, TableDataInfo *tdinfo); static void guessConstraintInheritance(TableInfo *tblinfo, int numTables); @@ -236,11 +233,8 @@ static char *convertRegProcReference(Archive *fout, const char *proc); static char *convertOperatorReference(Archive *fout, const char *opr); static char *convertTSFunction(Archive *fout, Oid funcOid); -static Oid findLastBuiltinOid_V71(Archive *fout, const char *); -static Oid findLastBuiltinOid_V70(Archive *fout); static void selectSourceSchema(Archive *fout, const char *schemaName); static char *getFormattedTypeName(Archive *fout, Oid oid, OidOptions opts); -static char *myFormatType(const char *typname, int32 typmod); static void getBlobs(Archive *fout); static void dumpBlob(Archive *fout, BlobInfo *binfo); static int dumpBlobs(Archive *fout, void *arg); @@ -633,10 +627,10 @@ main(int argc, char **argv) fout->verbose = g_verbose; /* - * We allow the server to be back to 7.0, and up to any minor release of + * We allow the server to be back to 8.0, and up to any minor release of * our own major version. (See also version check in pg_dumpall.c.) */ - fout->minRemoteVersion = 70000; + fout->minRemoteVersion = 80000; fout->maxRemoteVersion = (PG_VERSION_NUM / 100) * 100 + 99; fout->numWorkers = numWorkers; @@ -665,10 +659,8 @@ main(int argc, char **argv) /* Select the appropriate subquery to convert user IDs to names */ if (fout->remoteVersion >= 80100) username_subquery = "SELECT rolname FROM pg_catalog.pg_roles WHERE oid ="; - else if (fout->remoteVersion >= 70300) - username_subquery = "SELECT usename FROM pg_catalog.pg_user WHERE usesysid ="; else - username_subquery = "SELECT usename FROM pg_user WHERE usesysid ="; + username_subquery = "SELECT usename FROM pg_catalog.pg_user WHERE usesysid ="; /* check the version for the synchronized snapshots feature */ if (numWorkers > 1 && fout->remoteVersion < 90200 @@ -683,18 +675,6 @@ main(int argc, char **argv) exit_horribly(NULL, "Exported snapshots are not supported by this server version.\n"); - /* Find the last built-in OID, if needed */ - if (fout->remoteVersion < 70300) - { - if (fout->remoteVersion >= 70100) - g_last_builtin_oid = findLastBuiltinOid_V71(fout, - PQdb(GetConnection(fout))); - else - g_last_builtin_oid = findLastBuiltinOid_V70(fout); - if (g_verbose) - write_msg(NULL, "last built-in OID is %u\n", g_last_builtin_oid); - } - /* Expand schema selection patterns into OID lists */ if (schema_include_patterns.head != NULL) { @@ -774,16 +754,11 @@ main(int argc, char **argv) /* * Sort the objects into a safe dump order (no forward references). * - * In 7.3 or later, we can rely on dependency information to help us - * determine a safe order, so the initial sort is mostly for cosmetic - * purposes: we sort by name to ensure that logically identical schemas - * will dump identically. Before 7.3 we don't have dependencies and we - * use OID ordering as an (unreliable) guide to creation order. + * We rely on dependency information to help us determine a safe order, so + * the initial sort is mostly for cosmetic purposes: we sort by name to + * ensure that logically identical schemas will dump identically. */ - if (fout->remoteVersion >= 70300) - sortDumpableObjectsByTypeName(dobjs, numObjs); - else - sortDumpableObjectsByTypeOid(dobjs, numObjs); + sortDumpableObjectsByTypeName(dobjs, numObjs); /* If we do a parallel dump, we want the largest tables to go first */ if (archiveFormat == archDirectory && numWorkers > 1) @@ -1000,12 +975,12 @@ setup_connection(Archive *AH, const char *dumpencoding, ExecuteSqlStatement(AH, "SET INTERVALSTYLE = POSTGRES"); /* - * If supported, set extra_float_digits so that we can dump float data - * exactly (given correctly implemented float I/O code, anyway) + * Set extra_float_digits so that we can dump float data exactly (given + * correctly implemented float I/O code, anyway) */ if (AH->remoteVersion >= 90000) ExecuteSqlStatement(AH, "SET extra_float_digits TO 3"); - else if (AH->remoteVersion >= 70400) + else ExecuteSqlStatement(AH, "SET extra_float_digits TO 2"); /* @@ -1018,8 +993,7 @@ setup_connection(Archive *AH, const char *dumpencoding, /* * Disable timeouts if supported. */ - if (AH->remoteVersion >= 70300) - ExecuteSqlStatement(AH, "SET statement_timeout = 0"); + ExecuteSqlStatement(AH, "SET statement_timeout = 0"); if (AH->remoteVersion >= 90300) ExecuteSqlStatement(AH, "SET lock_timeout = 0"); if (AH->remoteVersion >= 90600) @@ -1065,16 +1039,12 @@ setup_connection(Archive *AH, const char *dumpencoding, "SET TRANSACTION ISOLATION LEVEL " "REPEATABLE READ, READ ONLY"); } - else if (AH->remoteVersion >= 70400) + else { - /* note: comma was not accepted in SET TRANSACTION before 8.0 */ ExecuteSqlStatement(AH, "SET TRANSACTION ISOLATION LEVEL " - "SERIALIZABLE READ ONLY"); + "SERIALIZABLE, READ ONLY"); } - else - ExecuteSqlStatement(AH, - "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"); /* * If user specified a snapshot to use, select that. In a parallel dump @@ -1190,9 +1160,6 @@ expand_schema_name_patterns(Archive *fout, if (patterns->head == NULL) return; /* nothing to do */ - if (fout->remoteVersion < 70300) - exit_horribly(NULL, "server version must be at least 7.3 to use schema selection switches\n"); - query = createPQExpBuffer(); /* @@ -1661,15 +1628,12 @@ dumpTableData_copy(Archive *fout, void *dcontext) selectSourceSchema(fout, tbinfo->dobj.namespace->dobj.name); /* - * If possible, specify the column list explicitly so that we have no - * possibility of retrieving data in the wrong column order. (The default - * column ordering of COPY will not be what we want in certain corner - * cases involving ADD COLUMN and inheritance.) + * Specify the column list explicitly so that we have no possibility of + * retrieving data in the wrong column order. (The default column + * ordering of COPY will not be what we want in certain corner cases + * involving ADD COLUMN and inheritance.) */ - if (fout->remoteVersion >= 70300) - column_list = fmtCopyColumnList(tbinfo, clistBuf); - else - column_list = ""; /* can't select columns in COPY */ + column_list = fmtCopyColumnList(tbinfo, clistBuf); if (oids && hasoids) { @@ -1829,22 +1793,11 @@ dumpTableData_insert(Archive *fout, void *dcontext) */ selectSourceSchema(fout, tbinfo->dobj.namespace->dobj.name); - if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(q, "DECLARE _pg_dump_cursor CURSOR FOR " - "SELECT * FROM ONLY %s", - fmtQualifiedId(fout->remoteVersion, - tbinfo->dobj.namespace->dobj.name, - classname)); - } - else - { - appendPQExpBuffer(q, "DECLARE _pg_dump_cursor CURSOR FOR " - "SELECT * FROM %s", - fmtQualifiedId(fout->remoteVersion, - tbinfo->dobj.namespace->dobj.name, - classname)); - } + appendPQExpBuffer(q, "DECLARE _pg_dump_cursor CURSOR FOR " + "SELECT * FROM ONLY %s", + fmtQualifiedId(fout->remoteVersion, + tbinfo->dobj.namespace->dobj.name, + classname)); if (tdinfo->filtercond) appendPQExpBuffer(q, " %s", tdinfo->filtercond); @@ -2480,7 +2433,7 @@ dumpDatabase(Archive *fout) username_subquery); appendStringLiteralAH(dbQry, datname, fout); } - else if (fout->remoteVersion >= 80000) + else { appendPQExpBuffer(dbQry, "SELECT tableoid, oid, " "(%s datdba) AS dba, " @@ -2492,34 +2445,6 @@ dumpDatabase(Archive *fout) username_subquery); appendStringLiteralAH(dbQry, datname, fout); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(dbQry, "SELECT tableoid, oid, " - "(%s datdba) AS dba, " - "pg_encoding_to_char(encoding) AS encoding, " - "NULL AS datcollate, NULL AS datctype, " - "0 AS datfrozenxid, 0 AS datminmxid, " - "NULL AS tablespace " - "FROM pg_database " - "WHERE datname = ", - username_subquery); - appendStringLiteralAH(dbQry, datname, fout); - } - else - { - appendPQExpBuffer(dbQry, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_database') AS tableoid, " - "oid, " - "(%s datdba) AS dba, " - "pg_encoding_to_char(encoding) AS encoding, " - "NULL AS datcollate, NULL AS datctype, " - "0 AS datfrozenxid, 0 AS datminmxid, " - "NULL AS tablespace " - "FROM pg_database " - "WHERE datname = ", - username_subquery); - appendStringLiteralAH(dbQry, datname, fout); - } res = ExecuteSqlQueryForSingleRow(fout, dbQry->data); @@ -2879,19 +2804,13 @@ getBlobs(Archive *fout) "NULL AS initrlomacl " " FROM pg_largeobject_metadata", username_subquery); - else if (fout->remoteVersion >= 70100) + else appendPQExpBufferStr(blobQry, "SELECT DISTINCT loid AS oid, " "NULL::name AS rolname, NULL::oid AS lomacl, " "NULL::oid AS rlomacl, NULL::oid AS initlomacl, " "NULL::oid AS initrlomacl " " FROM pg_largeobject"); - else - appendPQExpBufferStr(blobQry, - "SELECT oid, NULL AS rolname, NULL AS lomacl, " - "NULL AS rlomacl, NULL AS initlomacl, " - "NULL AS initrlomacl " - " FROM pg_class WHERE relkind = 'l'"); res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK); @@ -3031,10 +2950,8 @@ dumpBlobs(Archive *fout, void *arg) */ if (fout->remoteVersion >= 90000) blobQry = "DECLARE bloboid CURSOR FOR SELECT oid FROM pg_largeobject_metadata"; - else if (fout->remoteVersion >= 70100) - blobQry = "DECLARE bloboid CURSOR FOR SELECT DISTINCT loid FROM pg_largeobject"; else - blobQry = "DECLARE bloboid CURSOR FOR SELECT oid FROM pg_class WHERE relkind = 'l'"; + blobQry = "DECLARE bloboid CURSOR FOR SELECT DISTINCT loid FROM pg_largeobject"; ExecuteSqlStatement(fout, blobQry); @@ -3536,45 +3453,6 @@ getNamespaces(Archive *fout, int *numNamespaces) int i_initnspacl; int i_initrnspacl; - /* - * Before 7.3, there are no real namespaces; create two dummy entries, one - * for user stuff and one for system stuff. - */ - if (fout->remoteVersion < 70300) - { - nsinfo = (NamespaceInfo *) pg_malloc(2 * sizeof(NamespaceInfo)); - - nsinfo[0].dobj.objType = DO_NAMESPACE; - nsinfo[0].dobj.catId.tableoid = 0; - nsinfo[0].dobj.catId.oid = 0; - AssignDumpId(&nsinfo[0].dobj); - nsinfo[0].dobj.name = pg_strdup("public"); - nsinfo[0].rolname = pg_strdup(""); - nsinfo[0].nspacl = pg_strdup(""); - nsinfo[0].rnspacl = pg_strdup(""); - nsinfo[0].initnspacl = pg_strdup(""); - nsinfo[0].initrnspacl = pg_strdup(""); - - selectDumpableNamespace(&nsinfo[0], fout); - - nsinfo[1].dobj.objType = DO_NAMESPACE; - nsinfo[1].dobj.catId.tableoid = 0; - nsinfo[1].dobj.catId.oid = 1; - AssignDumpId(&nsinfo[1].dobj); - nsinfo[1].dobj.name = pg_strdup("pg_catalog"); - nsinfo[1].rolname = pg_strdup(""); - nsinfo[1].nspacl = pg_strdup(""); - nsinfo[1].rnspacl = pg_strdup(""); - nsinfo[1].initnspacl = pg_strdup(""); - nsinfo[1].initrnspacl = pg_strdup(""); - - selectDumpableNamespace(&nsinfo[1], fout); - - *numNamespaces = 2; - - return nsinfo; - } - query = createPQExpBuffer(); /* Make sure we are in proper schema */ @@ -3684,37 +3562,16 @@ getNamespaces(Archive *fout, int *numNamespaces) /* * findNamespace: - * given a namespace OID and an object OID, look up the info read by - * getNamespaces - * - * NB: for pre-7.3 source database, we use object OID to guess whether it's - * a system object or not. In 7.3 and later there is no guessing, and we - * don't use objoid at all. + * given a namespace OID, look up the info read by getNamespaces */ static NamespaceInfo * -findNamespace(Archive *fout, Oid nsoid, Oid objoid) +findNamespace(Archive *fout, Oid nsoid) { NamespaceInfo *nsinfo; - if (fout->remoteVersion >= 70300) - { - nsinfo = findNamespaceByOid(nsoid); - } - else - { - /* This code depends on the dummy objects set up by getNamespaces. */ - Oid i; - - if (objoid > g_last_builtin_oid) - i = 0; /* user object */ - else - i = 1; /* system object */ - nsinfo = findNamespaceByOid(i); - } - + nsinfo = findNamespaceByOid(nsoid); if (nsinfo == NULL) exit_horribly(NULL, "schema with OID %u does not exist\n", nsoid); - return nsinfo; } @@ -3932,7 +3789,7 @@ getTypes(Archive *fout, int *numTypes) "FROM pg_type", username_subquery); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT tableoid, oid, typname, " "typnamespace, NULL AS typacl, NULL as rtypacl, " @@ -3947,38 +3804,6 @@ getTypes(Archive *fout, int *numTypes) "FROM pg_type", username_subquery); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, typname, " - "0::oid AS typnamespace, NULL AS typacl, NULL as rtypacl, " - "NULL AS inittypacl, NULL AS initrtypacl, " - "(%s typowner) AS rolname, " - "typinput::oid AS typinput, " - "typoutput::oid AS typoutput, typelem, typrelid, " - "CASE WHEN typrelid = 0 THEN ' '::\"char\" " - "ELSE (SELECT relkind FROM pg_class WHERE oid = typrelid) END AS typrelkind, " - "typtype, typisdefined, " - "typname[0] = '_' AND typelem != 0 AS isarray " - "FROM pg_type", - username_subquery); - } - else - { - appendPQExpBuffer(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_type') AS tableoid, " - "oid, typname, " - "0::oid AS typnamespace, NULL AS typacl, NULL as rtypacl, " - "NULL AS inittypacl, NULL AS initrtypacl, " - "(%s typowner) AS rolname, " - "typinput::oid AS typinput, " - "typoutput::oid AS typoutput, typelem, typrelid, " - "CASE WHEN typrelid = 0 THEN ' '::\"char\" " - "ELSE (SELECT relkind FROM pg_class WHERE oid = typrelid) END AS typrelkind, " - "typtype, typisdefined, " - "typname[0] = '_' AND typelem != 0 AS isarray " - "FROM pg_type", - username_subquery); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -4013,8 +3838,7 @@ getTypes(Archive *fout, int *numTypes) tyinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_typname)); tyinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_typnamespace)), - tyinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_typnamespace))); tyinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); tyinfo[i].typacl = pg_strdup(PQgetvalue(res, i, i_typacl)); tyinfo[i].rtypacl = pg_strdup(PQgetvalue(res, i, i_rtypacl)); @@ -4083,48 +3907,6 @@ getTypes(Archive *fout, int *numTypes) * this is taken care of while sorting dependencies. */ stinfo->dobj.dump = DUMP_COMPONENT_NONE; - - /* - * However, if dumping from pre-7.3, there will be no dependency - * info so we have to fake it here. We only need to worry about - * typinput and typoutput since the other functions only exist - * post-7.3. - */ - if (fout->remoteVersion < 70300) - { - Oid typinput; - Oid typoutput; - FuncInfo *funcInfo; - - typinput = atooid(PQgetvalue(res, i, i_typinput)); - typoutput = atooid(PQgetvalue(res, i, i_typoutput)); - - funcInfo = findFuncByOid(typinput); - if (funcInfo && funcInfo->dobj.dump & DUMP_COMPONENT_DEFINITION) - { - /* base type depends on function */ - addObjectDependency(&tyinfo[i].dobj, - funcInfo->dobj.dumpId); - /* function depends on shell type */ - addObjectDependency(&funcInfo->dobj, - stinfo->dobj.dumpId); - /* mark shell type as to be dumped */ - stinfo->dobj.dump = DUMP_COMPONENT_ALL; - } - - funcInfo = findFuncByOid(typoutput); - if (funcInfo && funcInfo->dobj.dump & DUMP_COMPONENT_DEFINITION) - { - /* base type depends on function */ - addObjectDependency(&tyinfo[i].dobj, - funcInfo->dobj.dumpId); - /* function depends on shell type */ - addObjectDependency(&funcInfo->dobj, - stinfo->dobj.dumpId); - /* mark shell type as to be dumped */ - stinfo->dobj.dump = DUMP_COMPONENT_ALL; - } - } } if (strlen(tyinfo[i].rolname) == 0) @@ -4172,38 +3954,13 @@ getOperators(Archive *fout, int *numOprs) /* Make sure we are in proper schema */ selectSourceSchema(fout, "pg_catalog"); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, oprname, " - "oprnamespace, " - "(%s oprowner) AS rolname, " - "oprkind, " - "oprcode::oid AS oprcode " - "FROM pg_operator", - username_subquery); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, oprname, " - "0::oid AS oprnamespace, " - "(%s oprowner) AS rolname, " - "oprkind, " - "oprcode::oid AS oprcode " - "FROM pg_operator", - username_subquery); - } - else - { - appendPQExpBuffer(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_operator') AS tableoid, " - "oid, oprname, " - "0::oid AS oprnamespace, " - "(%s oprowner) AS rolname, " - "oprkind, " - "oprcode::oid AS oprcode " - "FROM pg_operator", - username_subquery); - } + appendPQExpBuffer(query, "SELECT tableoid, oid, oprname, " + "oprnamespace, " + "(%s oprowner) AS rolname, " + "oprkind, " + "oprcode::oid AS oprcode " + "FROM pg_operator", + username_subquery); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -4229,8 +3986,7 @@ getOperators(Archive *fout, int *numOprs) oprinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oprname)); oprinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_oprnamespace)), - oprinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_oprnamespace))); oprinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); oprinfo[i].oprkind = (PQgetvalue(res, i, i_oprkind))[0]; oprinfo[i].oprcode = atooid(PQgetvalue(res, i, i_oprcode)); @@ -4319,8 +4075,7 @@ getCollations(Archive *fout, int *numCollations) collinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_collname)); collinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_collnamespace)), - collinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_collnamespace))); collinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -4358,13 +4113,6 @@ getConversions(Archive *fout, int *numConversions) int i_connamespace; int i_rolname; - /* Conversions didn't exist pre-7.3 */ - if (fout->remoteVersion < 70300) - { - *numConversions = 0; - return NULL; - } - query = createPQExpBuffer(); /* @@ -4403,8 +4151,7 @@ getConversions(Archive *fout, int *numConversions) convinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_conname)); convinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_connamespace)), - convinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_connamespace))); convinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -4527,30 +4274,11 @@ getOpclasses(Archive *fout, int *numOpclasses) /* Make sure we are in proper schema */ selectSourceSchema(fout, "pg_catalog"); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, opcname, " - "opcnamespace, " - "(%s opcowner) AS rolname " - "FROM pg_opclass", - username_subquery); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBufferStr(query, "SELECT tableoid, oid, opcname, " - "0::oid AS opcnamespace, " - "''::name AS rolname " - "FROM pg_opclass"); - } - else - { - appendPQExpBufferStr(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_opclass') AS tableoid, " - "oid, opcname, " - "0::oid AS opcnamespace, " - "''::name AS rolname " - "FROM pg_opclass"); - } + appendPQExpBuffer(query, "SELECT tableoid, oid, opcname, " + "opcnamespace, " + "(%s opcowner) AS rolname " + "FROM pg_opclass", + username_subquery); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -4574,8 +4302,7 @@ getOpclasses(Archive *fout, int *numOpclasses) opcinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_opcname)); opcinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_opcnamespace)), - opcinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_opcnamespace))); opcinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -4584,12 +4311,9 @@ getOpclasses(Archive *fout, int *numOpclasses) /* Op Classes do not currently have ACLs. */ opcinfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL; - if (fout->remoteVersion >= 70300) - { - if (strlen(opcinfo[i].rolname) == 0) - write_msg(NULL, "WARNING: owner of operator class \"%s\" appears to be invalid\n", - opcinfo[i].dobj.name); - } + if (strlen(opcinfo[i].rolname) == 0) + write_msg(NULL, "WARNING: owner of operator class \"%s\" appears to be invalid\n", + opcinfo[i].dobj.name); } PQclear(res); @@ -4665,8 +4389,7 @@ getOpfamilies(Archive *fout, int *numOpfamilies) opfinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_opfname)); opfinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_opfnamespace)), - opfinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_opfnamespace))); opfinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -4675,12 +4398,9 @@ getOpfamilies(Archive *fout, int *numOpfamilies) /* Extensions do not currently have ACLs. */ opfinfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL; - if (fout->remoteVersion >= 70300) - { - if (strlen(opfinfo[i].rolname) == 0) - write_msg(NULL, "WARNING: owner of operator family \"%s\" appears to be invalid\n", - opfinfo[i].dobj.name); - } + if (strlen(opfinfo[i].rolname) == 0) + write_msg(NULL, "WARNING: owner of operator family \"%s\" appears to be invalid\n", + opfinfo[i].dobj.name); } PQclear(res); @@ -4798,7 +4518,7 @@ getAggregates(Archive *fout, int *numAggs) "deptype = 'e')"); appendPQExpBufferChar(query, ')'); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT tableoid, oid, proname AS aggname, " "pronamespace AS aggnamespace, " @@ -4814,38 +4534,6 @@ getAggregates(Archive *fout, int *numAggs) "(SELECT oid FROM pg_namespace WHERE nspname = 'pg_catalog')", username_subquery); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, aggname, " - "0::oid AS aggnamespace, " - "CASE WHEN aggbasetype = 0 THEN 0 ELSE 1 END AS pronargs, " - "aggbasetype AS proargtypes, " - "(%s aggowner) AS rolname, " - "NULL AS aggacl, " - "NULL AS raggacl, " - "NULL AS initaggacl, NULL AS initraggacl " - "FROM pg_aggregate " - "where oid > '%u'::oid", - username_subquery, - g_last_builtin_oid); - } - else - { - appendPQExpBuffer(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_aggregate') AS tableoid, " - "oid, aggname, " - "0::oid AS aggnamespace, " - "CASE WHEN aggbasetype = 0 THEN 0 ELSE 1 END AS pronargs, " - "aggbasetype AS proargtypes, " - "(%s aggowner) AS rolname, " - "NULL AS aggacl, " - "NULL AS raggacl, " - "NULL AS initaggacl, NULL AS initraggacl " - "FROM pg_aggregate " - "where oid > '%u'::oid", - username_subquery, - g_last_builtin_oid); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -4875,8 +4563,7 @@ getAggregates(Archive *fout, int *numAggs) agginfo[i].aggfn.dobj.name = pg_strdup(PQgetvalue(res, i, i_aggname)); agginfo[i].aggfn.dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_aggnamespace)), - agginfo[i].aggfn.dobj.catId.oid); + atooid(PQgetvalue(res, i, i_aggnamespace))); agginfo[i].aggfn.rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); if (strlen(agginfo[i].aggfn.rolname) == 0) write_msg(NULL, "WARNING: owner of aggregate function \"%s\" appears to be invalid\n", @@ -4893,13 +4580,9 @@ getAggregates(Archive *fout, int *numAggs) else { agginfo[i].aggfn.argtypes = (Oid *) pg_malloc(agginfo[i].aggfn.nargs * sizeof(Oid)); - if (fout->remoteVersion >= 70300) - parseOidArray(PQgetvalue(res, i, i_proargtypes), - agginfo[i].aggfn.argtypes, - agginfo[i].aggfn.nargs); - else - /* it's just aggbasetype */ - agginfo[i].aggfn.argtypes[0] = atooid(PQgetvalue(res, i, i_proargtypes)); + parseOidArray(PQgetvalue(res, i, i_proargtypes), + agginfo[i].aggfn.argtypes, + agginfo[i].aggfn.nargs); } /* Decide whether we want to dump it */ @@ -5025,7 +4708,7 @@ getFuncs(Archive *fout, int *numFuncs) destroyPQExpBuffer(initacl_subquery); destroyPQExpBuffer(initracl_subquery); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT tableoid, oid, proname, prolang, " @@ -5056,39 +4739,6 @@ getFuncs(Archive *fout, int *numFuncs) "deptype = 'e')"); appendPQExpBufferChar(query, ')'); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, - "SELECT tableoid, oid, proname, prolang, " - "pronargs, proargtypes, prorettype, " - "NULL AS proacl, " - "NULL AS rproacl, " - "NULL as initproacl, NULL AS initrproacl, " - "0::oid AS pronamespace, " - "(%s proowner) AS rolname " - "FROM pg_proc " - "WHERE pg_proc.oid > '%u'::oid", - username_subquery, - g_last_builtin_oid); - } - else - { - appendPQExpBuffer(query, - "SELECT " - "(SELECT oid FROM pg_class " - " WHERE relname = 'pg_proc') AS tableoid, " - "oid, proname, prolang, " - "pronargs, proargtypes, prorettype, " - "NULL AS proacl, " - "NULL AS rproacl, " - "NULL as initproacl, NULL AS initrproacl, " - "0::oid AS pronamespace, " - "(%s proowner) AS rolname " - "FROM pg_proc " - "where pg_proc.oid > '%u'::oid", - username_subquery, - g_last_builtin_oid); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -5121,8 +4771,7 @@ getFuncs(Archive *fout, int *numFuncs) finfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_proname)); finfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_pronamespace)), - finfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_pronamespace))); finfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); finfo[i].lang = atooid(PQgetvalue(res, i, i_prolang)); finfo[i].prorettype = atooid(PQgetvalue(res, i, i_prorettype)); @@ -5645,7 +5294,7 @@ getTables(Archive *fout, int *numTables) RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, RELKIND_COMPOSITE_TYPE); } - else if (fout->remoteVersion >= 80000) + else { /* * Left join to pick up dependency info linking sequences to their @@ -5686,153 +5335,6 @@ getTables(Archive *fout, int *numTables) RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, RELKIND_COMPOSITE_TYPE); } - else if (fout->remoteVersion >= 70300) - { - /* - * Left join to pick up dependency info linking sequences to their - * owning column, if any - */ - appendPQExpBuffer(query, - "SELECT c.tableoid, c.oid, relname, " - "relacl, NULL as rrelacl, " - "NULL AS initrelacl, NULL AS initrrelacl, " - "relkind, relnamespace, " - "(%s relowner) AS rolname, " - "relchecks, (reltriggers <> 0) AS relhastriggers, " - "relhasindex, relhasrules, relhasoids, " - "'f'::bool AS relrowsecurity, " - "'f'::bool AS relforcerowsecurity, " - "0 AS relfrozenxid, 0 AS relminmxid," - "0 AS toid, " - "0 AS tfrozenxid, 0 AS tminmxid," - "'p' AS relpersistence, 't' as relispopulated, " - "'d' AS relreplident, relpages, " - "NULL AS reloftype, " - "d.refobjid AS owning_tab, " - "d.refobjsubid AS owning_col, " - "NULL AS reltablespace, " - "NULL AS reloptions, " - "NULL AS toast_reloptions, " - "NULL AS changed_acl " - "FROM pg_class c " - "LEFT JOIN pg_depend d ON " - "(c.relkind = '%c' AND " - "d.classid = c.tableoid AND d.objid = c.oid AND " - "d.objsubid = 0 AND " - "d.refclassid = c.tableoid AND d.deptype = 'i') " - "WHERE relkind IN ('%c', '%c', '%c', '%c') " - "ORDER BY c.oid", - username_subquery, - RELKIND_SEQUENCE, - RELKIND_RELATION, RELKIND_SEQUENCE, - RELKIND_VIEW, RELKIND_COMPOSITE_TYPE); - } - else if (fout->remoteVersion >= 70200) - { - appendPQExpBuffer(query, - "SELECT tableoid, oid, relname, relacl, " - "NULL as rrelacl, " - "NULL AS initrelacl, NULL AS initrrelacl, " - "relkind, " - "0::oid AS relnamespace, " - "(%s relowner) AS rolname, " - "relchecks, (reltriggers <> 0) AS relhastriggers, " - "relhasindex, relhasrules, relhasoids, " - "'f'::bool AS relrowsecurity, " - "'f'::bool AS relforcerowsecurity, " - "0 AS relfrozenxid, 0 AS relminmxid," - "0 AS toid, " - "0 AS tfrozenxid, 0 AS tminmxid," - "'p' AS relpersistence, 't' as relispopulated, " - "'d' AS relreplident, relpages, " - "NULL AS reloftype, " - "NULL::oid AS owning_tab, " - "NULL::int4 AS owning_col, " - "NULL AS reltablespace, " - "NULL AS reloptions, " - "NULL AS toast_reloptions, " - "NULL AS changed_acl " - "FROM pg_class " - "WHERE relkind IN ('%c', '%c', '%c') " - "ORDER BY oid", - username_subquery, - RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW); - } - else if (fout->remoteVersion >= 70100) - { - /* all tables have oids in 7.1 */ - appendPQExpBuffer(query, - "SELECT tableoid, oid, relname, relacl, " - "NULL as rrelacl, " - "NULL AS initrelacl, NULL AS initrrelacl, " - "relkind, " - "0::oid AS relnamespace, " - "(%s relowner) AS rolname, " - "relchecks, (reltriggers <> 0) AS relhastriggers, " - "relhasindex, relhasrules, " - "'t'::bool AS relhasoids, " - "'f'::bool AS relrowsecurity, " - "'f'::bool AS relforcerowsecurity, " - "0 AS relfrozenxid, 0 AS relminmxid," - "0 AS toid, " - "0 AS tfrozenxid, 0 AS tminmxid," - "'p' AS relpersistence, 't' as relispopulated, " - "'d' AS relreplident, relpages, " - "NULL AS reloftype, " - "NULL::oid AS owning_tab, " - "NULL::int4 AS owning_col, " - "NULL AS reltablespace, " - "NULL AS reloptions, " - "NULL AS toast_reloptions, " - "NULL AS changed_acl " - "FROM pg_class " - "WHERE relkind IN ('%c', '%c', '%c') " - "ORDER BY oid", - username_subquery, - RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW); - } - else - { - /* - * Before 7.1, view relkind was not set to 'v', so we must check if we - * have a view by looking for a rule in pg_rewrite. - */ - appendPQExpBuffer(query, - "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_class') AS tableoid, " - "oid, relname, relacl, NULL as rrelacl, " - "NULL AS initrelacl, NULL AS initrrelacl, " - "CASE WHEN relhasrules and relkind = 'r' " - " and EXISTS(SELECT rulename FROM pg_rewrite r WHERE " - " r.ev_class = c.oid AND r.ev_type = '1') " - "THEN '%c'::\"char\" " - "ELSE relkind END AS relkind," - "0::oid AS relnamespace, " - "(%s relowner) AS rolname, " - "relchecks, (reltriggers <> 0) AS relhastriggers, " - "relhasindex, relhasrules, " - "'t'::bool AS relhasoids, " - "'f'::bool AS relrowsecurity, " - "'f'::bool AS relforcerowsecurity, " - "0 AS relfrozenxid, 0 AS relminmxid," - "0 AS toid, " - "0 AS tfrozenxid, 0 AS tminmxid," - "'p' AS relpersistence, 't' as relispopulated, " - "'d' AS relreplident, 0 AS relpages, " - "NULL AS reloftype, " - "NULL::oid AS owning_tab, " - "NULL::int4 AS owning_col, " - "NULL AS reltablespace, " - "NULL AS reloptions, " - "NULL AS toast_reloptions, " - "NULL AS changed_acl " - "FROM pg_class c " - "WHERE relkind IN ('%c', '%c') " - "ORDER BY oid", - RELKIND_VIEW, - username_subquery, - RELKIND_RELATION, RELKIND_SEQUENCE); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -5886,7 +5388,7 @@ getTables(Archive *fout, int *numTables) i_reloftype = PQfnumber(res, "reloftype"); i_changed_acl = PQfnumber(res, "changed_acl"); - if (dopt->lockWaitTimeout && fout->remoteVersion >= 70300) + if (dopt->lockWaitTimeout) { /* * Arrange to fail instead of waiting forever for a table lock. @@ -5910,8 +5412,7 @@ getTables(Archive *fout, int *numTables) tblinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_relname)); tblinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_relnamespace)), - tblinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_relnamespace))); tblinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); tblinfo[i].relacl = pg_strdup(PQgetvalue(res, i, i_relacl)); tblinfo[i].rrelacl = pg_strdup(PQgetvalue(res, i, i_rrelacl)); @@ -6017,7 +5518,7 @@ getTables(Archive *fout, int *numTables) tblinfo[i].dobj.name); } - if (dopt->lockWaitTimeout && fout->remoteVersion >= 70300) + if (dopt->lockWaitTimeout) { ExecuteSqlStatement(fout, "SET statement_timeout = 0"); } @@ -6290,7 +5791,7 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "ORDER BY indexname", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 80000) + else { appendPQExpBuffer(query, "SELECT t.tableoid, t.oid, " @@ -6319,87 +5820,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "ORDER BY indexname", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " - "t.relnatts AS indnkeys, " - "i.indkey, i.indisclustered, " - "false AS indisreplident, t.relpages, " - "c.contype, c.conname, " - "c.condeferrable, c.condeferred, " - "c.tableoid AS contableoid, " - "c.oid AS conoid, " - "null AS condef, " - "NULL AS tablespace, " - "null AS indreloptions " - "FROM pg_catalog.pg_index i " - "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " - "LEFT JOIN pg_catalog.pg_depend d " - "ON (d.classid = t.tableoid " - "AND d.objid = t.oid " - "AND d.deptype = 'i') " - "LEFT JOIN pg_catalog.pg_constraint c " - "ON (d.refclassid = c.tableoid " - "AND d.refobjid = c.oid) " - "WHERE i.indrelid = '%u'::pg_catalog.oid " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "pg_get_indexdef(i.indexrelid) AS indexdef, " - "t.relnatts AS indnkeys, " - "i.indkey, false AS indisclustered, " - "false AS indisreplident, t.relpages, " - "CASE WHEN i.indisprimary THEN 'p'::char " - "ELSE '0'::char END AS contype, " - "t.relname AS conname, " - "false AS condeferrable, " - "false AS condeferred, " - "0::oid AS contableoid, " - "t.oid AS conoid, " - "null AS condef, " - "NULL AS tablespace, " - "null AS indreloptions " - "FROM pg_index i, pg_class t " - "WHERE t.oid = i.indexrelid " - "AND i.indrelid = '%u'::oid " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, - "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_class') AS tableoid, " - "t.oid, " - "t.relname AS indexname, " - "pg_get_indexdef(i.indexrelid) AS indexdef, " - "t.relnatts AS indnkeys, " - "i.indkey, false AS indisclustered, " - "false AS indisreplident, t.relpages, " - "CASE WHEN i.indisprimary THEN 'p'::char " - "ELSE '0'::char END AS contype, " - "t.relname AS conname, " - "false AS condeferrable, " - "false AS condeferred, " - "0::oid AS contableoid, " - "t.oid AS conoid, " - "null AS condef, " - "NULL AS tablespace, " - "null AS indreloptions " - "FROM pg_index i, pg_class t " - "WHERE t.oid = i.indexrelid " - "AND i.indrelid = '%u'::oid " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -6442,19 +5862,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) indxinfo[j].indnkeys = atoi(PQgetvalue(res, j, i_indnkeys)); indxinfo[j].tablespace = pg_strdup(PQgetvalue(res, j, i_tablespace)); indxinfo[j].indreloptions = pg_strdup(PQgetvalue(res, j, i_indreloptions)); - - /* - * In pre-7.4 releases, indkeys may contain more entries than - * indnkeys says (since indnkeys will be 1 for a functional - * index). We don't actually care about this case since we don't - * examine indkeys except for indexes associated with PRIMARY and - * UNIQUE constraints, which are never functional indexes. But we - * have to allocate enough space to keep parseOidArray from - * complaining. - */ - indxinfo[j].indkeys = (Oid *) pg_malloc(INDEX_MAX_KEYS * sizeof(Oid)); + indxinfo[j].indkeys = (Oid *) pg_malloc(indxinfo[j].indnkeys * sizeof(Oid)); parseOidArray(PQgetvalue(res, j, i_indkey), - indxinfo[j].indkeys, INDEX_MAX_KEYS); + indxinfo[j].indkeys, indxinfo[j].indnkeys); indxinfo[j].indisclustered = (PQgetvalue(res, j, i_indisclustered)[0] == 't'); indxinfo[j].indisreplident = (PQgetvalue(res, j, i_indisreplident)[0] == 't'); indxinfo[j].relpages = atoi(PQgetvalue(res, j, i_relpages)); @@ -6465,9 +5875,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) /* * If we found a constraint matching the index, create an * entry for it. - * - * In a pre-7.3 database, we take this path iff the index was - * marked indisprimary. */ constrinfo[j].dobj.objType = DO_CONSTRAINT; constrinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_contableoid)); @@ -6490,10 +5897,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) constrinfo[j].separate = true; indxinfo[j].indexconstraint = constrinfo[j].dobj.dumpId; - - /* If pre-7.3 DB, better make sure table comes first */ - addObjectDependency(&constrinfo[j].dobj, - tbinfo->dobj.dumpId); } else { @@ -6532,10 +5935,6 @@ getConstraints(Archive *fout, TableInfo tblinfo[], int numTables) i_condef; int ntups; - /* pg_constraint was created in 7.3, so nothing to do if older */ - if (fout->remoteVersion < 70300) - return; - query = createPQExpBuffer(); for (i = 0; i < numTables; i++) @@ -6621,10 +6020,6 @@ getDomainConstraints(Archive *fout, TypeInfo *tyinfo) i_consrc; int ntups; - /* pg_constraint was created in 7.3, so nothing to do if older */ - if (fout->remoteVersion < 70300) - return; - /* * select appropriate schema to ensure names in constraint are properly * qualified @@ -6642,17 +6037,9 @@ getDomainConstraints(Archive *fout, TypeInfo *tyinfo) "ORDER BY conname", tyinfo->dobj.catId.oid); - else if (fout->remoteVersion >= 70400) - appendPQExpBuffer(query, "SELECT tableoid, oid, conname, " - "pg_catalog.pg_get_constraintdef(oid) AS consrc, " - "true as convalidated " - "FROM pg_catalog.pg_constraint " - "WHERE contypid = '%u'::pg_catalog.oid " - "ORDER BY conname", - tyinfo->dobj.catId.oid); else appendPQExpBuffer(query, "SELECT tableoid, oid, conname, " - "'CHECK (' || consrc || ')' AS consrc, " + "pg_catalog.pg_get_constraintdef(oid) AS consrc, " "true as convalidated " "FROM pg_catalog.pg_constraint " "WHERE contypid = '%u'::pg_catalog.oid " @@ -6745,20 +6132,10 @@ getRules(Archive *fout, int *numRules) "FROM pg_rewrite " "ORDER BY oid"); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBufferStr(query, "SELECT " - "tableoid, oid, rulename, " - "ev_class AS ruletable, ev_type, is_instead, " - "'O'::char AS ev_enabled " - "FROM pg_rewrite " - "ORDER BY oid"); - } else { appendPQExpBufferStr(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_rewrite') AS tableoid, " - "oid, rulename, " + "tableoid, oid, rulename, " "ev_class AS ruletable, ev_type, is_instead, " "'O'::char AS ev_enabled " "FROM pg_rewrite " @@ -6931,7 +6308,7 @@ getTriggers(Archive *fout, TableInfo tblinfo[], int numTables) "AND tgconstraint = 0", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { /* * We ignore triggers that are tied to a foreign-key constraint, @@ -6954,34 +6331,7 @@ getTriggers(Archive *fout, TableInfo tblinfo[], int numTables) " WHERE d.classid = t.tableoid AND d.objid = t.oid AND d.deptype = 'i' AND c.contype = 'f'))", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, - "SELECT tgname, tgfoid::regproc AS tgfname, " - "tgtype, tgnargs, tgargs, tgenabled, " - "tgisconstraint, tgconstrname, tgdeferrable, " - "tgconstrrelid, tginitdeferred, tableoid, oid, " - "(SELECT relname FROM pg_class WHERE oid = tgconstrrelid) " - " AS tgconstrrelname " - "FROM pg_trigger " - "WHERE tgrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, - "SELECT tgname, tgfoid::regproc AS tgfname, " - "tgtype, tgnargs, tgargs, tgenabled, " - "tgisconstraint, tgconstrname, tgdeferrable, " - "tgconstrrelid, tginitdeferred, " - "(SELECT oid FROM pg_class WHERE relname = 'pg_trigger') AS tableoid, " - "oid, " - "(SELECT relname FROM pg_class WHERE oid = tgconstrrelid) " - " AS tgconstrrelname " - "FROM pg_trigger " - "WHERE tgrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); ntups = PQntuples(res); @@ -7278,7 +6628,7 @@ getProcLangs(Archive *fout, int *numProcLangs) "ORDER BY oid", username_subquery); } - else if (fout->remoteVersion >= 70400) + else { /* Languages are owned by the bootstrap superuser, sysid 1 */ appendPQExpBuffer(query, "SELECT tableoid, oid, " @@ -7292,45 +6642,6 @@ getProcLangs(Archive *fout, int *numProcLangs) "ORDER BY oid", username_subquery); } - else if (fout->remoteVersion >= 70300) - { - /* No clear notion of an owner at all before 7.4 ... */ - appendPQExpBuffer(query, "SELECT tableoid, oid, " - "lanname, lanpltrusted, lanplcallfoid, " - "0 AS laninline, lanvalidator, lanacl, " - "NULL AS rlanacl, " - "NULL AS initlanacl, NULL AS initrlanacl, " - "NULL AS lanowner " - "FROM pg_language " - "WHERE lanispl " - "ORDER BY oid"); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, " - "lanname, lanpltrusted, lanplcallfoid, " - "0 AS laninline, 0 AS lanvalidator, NULL AS lanacl, " - "NULL AS rlanacl, " - "NULL AS initlanacl, NULL AS initrlanacl, " - "NULL AS lanowner " - "FROM pg_language " - "WHERE lanispl " - "ORDER BY oid"); - } - else - { - appendPQExpBuffer(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_language') AS tableoid, " - "oid, " - "lanname, lanpltrusted, lanplcallfoid, " - "0 AS laninline, 0 AS lanvalidator, NULL AS lanacl, " - "NULL AS rlanacl, " - "NULL AS initlanacl, NULL AS initrlanacl, " - "NULL AS lanowner " - "FROM pg_language " - "WHERE lanispl " - "ORDER BY oid"); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -7379,20 +6690,6 @@ getProcLangs(Archive *fout, int *numProcLangs) PQgetisnull(res, i, i_initlanacl) && PQgetisnull(res, i, i_initrlanacl)) planginfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL; - - if (fout->remoteVersion < 70300) - { - /* - * We need to make a dependency to ensure the function will be - * dumped first. (In 7.3 and later the regular dependency - * mechanism will handle this for us.) - */ - FuncInfo *funcInfo = findFuncByOid(planginfo[i].lanplcallfoid); - - if (funcInfo) - addObjectDependency(&planginfo[i].dobj, - funcInfo->dobj.dumpId); - } } PQclear(res); @@ -7434,25 +6731,13 @@ getCasts(Archive *fout, int *numCasts) "castmethod " "FROM pg_cast ORDER BY 3,4"); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBufferStr(query, "SELECT tableoid, oid, " "castsource, casttarget, castfunc, castcontext, " "CASE WHEN castfunc = 0 THEN 'b' ELSE 'f' END AS castmethod " "FROM pg_cast ORDER BY 3,4"); } - else - { - appendPQExpBufferStr(query, "SELECT 0 AS tableoid, p.oid, " - "t1.oid AS castsource, t2.oid AS casttarget, " - "p.oid AS castfunc, 'e' AS castcontext, " - "'f' AS castmethod " - "FROM pg_type t1, pg_type t2, pg_proc p " - "WHERE p.pronargs = 1 AND " - "p.proargtypes[0] = t1.oid AND " - "p.prorettype = t2.oid AND p.proname = t2.typname " - "ORDER BY 3,4"); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -7499,22 +6784,6 @@ getCasts(Archive *fout, int *numCasts) sTypeInfo->dobj.name, tTypeInfo->dobj.name); castinfo[i].dobj.name = namebuf.data; - if (fout->remoteVersion < 70300 && - OidIsValid(castinfo[i].castfunc)) - { - /* - * We need to make a dependency to ensure the function will be - * dumped first. (In 7.3 and later the regular dependency - * mechanism handles this for us.) - */ - FuncInfo *funcInfo; - - funcInfo = findFuncByOid(castinfo[i].castfunc); - if (funcInfo) - addObjectDependency(&castinfo[i].dobj, - funcInfo->dobj.dumpId); - } - /* Decide whether we want to dump it */ selectDumpableCast(&(castinfo[i]), fout); @@ -7701,10 +6970,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) /* * we must read the attribute names in attribute number order! because - * we will use the attnum to index into the attnames array later. We - * actually ask to order by "attrelid, attnum" because (at least up to - * 7.3) the planner is not smart enough to realize it needn't re-sort - * the output of an indexscan on pg_attribute_relid_attnum_index. + * we will use the attnum to index into the attnames array later. */ if (g_verbose) write_msg(NULL, "finding the columns and types of table \"%s.%s\"\n", @@ -7736,7 +7002,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2 " - "ORDER BY a.attrelid, a.attnum", + "ORDER BY a.attnum", tbinfo->dobj.catId.oid); } else if (fout->remoteVersion >= 90100) @@ -7760,7 +7026,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2 " - "ORDER BY a.attrelid, a.attnum", + "ORDER BY a.attnum", tbinfo->dobj.catId.oid); } else if (fout->remoteVersion >= 90000) @@ -7778,10 +7044,10 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2 " - "ORDER BY a.attrelid, a.attnum", + "ORDER BY a.attnum", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { /* need left join here to not fail on dropped columns ... */ appendPQExpBuffer(q, "SELECT a.attnum, a.attname, a.atttypmod, " @@ -7795,50 +7061,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2 " - "ORDER BY a.attrelid, a.attnum", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - /* - * attstattarget doesn't exist in 7.1. It does exist in 7.2, but - * we don't dump it because we can't tell whether it's been - * explicitly set or was just a default. - * - * attislocal doesn't exist before 7.3, either; in older databases - * we assume it's TRUE, else we'd fail to dump non-inherited atts. - */ - appendPQExpBuffer(q, "SELECT a.attnum, a.attname, a.atttypmod, " - "-1 AS attstattarget, a.attstorage, " - "t.typstorage, a.attnotnull, a.atthasdef, " - "false AS attisdropped, a.attlen, " - "a.attalign, true AS attislocal, " - "format_type(t.oid,a.atttypmod) AS atttypname, " - "'' AS attoptions, 0 AS attcollation, " - "NULL AS attfdwoptions " - "FROM pg_attribute a LEFT JOIN pg_type t " - "ON a.atttypid = t.oid " - "WHERE a.attrelid = '%u'::oid " - "AND a.attnum > 0::int2 " - "ORDER BY a.attrelid, a.attnum", - tbinfo->dobj.catId.oid); - } - else - { - /* format_type not available before 7.1 */ - appendPQExpBuffer(q, "SELECT attnum, attname, atttypmod, " - "-1 AS attstattarget, " - "attstorage, attstorage AS typstorage, " - "attnotnull, atthasdef, false AS attisdropped, " - "attlen, attalign, " - "true AS attislocal, " - "(SELECT typname FROM pg_type WHERE oid = atttypid) AS atttypname, " - "'' AS attoptions, 0 AS attcollation, " - "NULL AS attfdwoptions " - "FROM pg_attribute a " - "WHERE attrelid = '%u'::oid " - "AND attnum > 0::int2 " - "ORDER BY attrelid, attnum", + "ORDER BY a.attnum", tbinfo->dobj.catId.oid); } @@ -7924,42 +7147,12 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->dobj.namespace->dobj.name, tbinfo->dobj.name); - resetPQExpBuffer(q); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(q, "SELECT tableoid, oid, adnum, " + printfPQExpBuffer(q, "SELECT tableoid, oid, adnum, " "pg_catalog.pg_get_expr(adbin, adrelid) AS adsrc " - "FROM pg_catalog.pg_attrdef " - "WHERE adrelid = '%u'::pg_catalog.oid", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70200) - { - /* 7.2 did not have OIDs in pg_attrdef */ - appendPQExpBuffer(q, "SELECT tableoid, 0 AS oid, adnum, " - "pg_get_expr(adbin, adrelid) AS adsrc " - "FROM pg_attrdef " - "WHERE adrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - /* no pg_get_expr, so must rely on adsrc */ - appendPQExpBuffer(q, "SELECT tableoid, oid, adnum, adsrc " - "FROM pg_attrdef " - "WHERE adrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } - else - { - /* no pg_get_expr, no tableoid either */ - appendPQExpBuffer(q, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_attrdef') AS tableoid, " - "oid, adnum, adsrc " - "FROM pg_attrdef " - "WHERE adrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } + "FROM pg_catalog.pg_attrdef " + "WHERE adrelid = '%u'::pg_catalog.oid", + tbinfo->dobj.catId.oid); + res = ExecuteSqlQuery(fout, q->data, PGRES_TUPLES_OK); numDefaults = PQntuples(res); @@ -8005,17 +7198,11 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) if (tbinfo->relkind == RELKIND_VIEW) { attrdefs[j].separate = true; - /* needed in case pre-7.3 DB: */ - addObjectDependency(&attrdefs[j].dobj, - tbinfo->dobj.dumpId); } else if (!shouldPrintColumn(dopt, tbinfo, adnum - 1)) { /* column will be suppressed, print default separately */ attrdefs[j].separate = true; - /* needed in case pre-7.3 DB: */ - addObjectDependency(&attrdefs[j].dobj, - tbinfo->dobj.dumpId); } else { @@ -8077,7 +7264,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ORDER BY conname", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70400) + else { appendPQExpBuffer(q, "SELECT tableoid, oid, conname, " "pg_catalog.pg_get_constraintdef(oid) AS consrc, " @@ -8088,54 +7275,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ORDER BY conname", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) - { - /* no pg_get_constraintdef, must use consrc */ - appendPQExpBuffer(q, "SELECT tableoid, oid, conname, " - "'CHECK (' || consrc || ')' AS consrc, " - "true AS conislocal, true AS convalidated " - "FROM pg_catalog.pg_constraint " - "WHERE conrelid = '%u'::pg_catalog.oid " - " AND contype = 'c' " - "ORDER BY conname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70200) - { - /* 7.2 did not have OIDs in pg_relcheck */ - appendPQExpBuffer(q, "SELECT tableoid, 0 AS oid, " - "rcname AS conname, " - "'CHECK (' || rcsrc || ')' AS consrc, " - "true AS conislocal, true AS convalidated " - "FROM pg_relcheck " - "WHERE rcrelid = '%u'::oid " - "ORDER BY rcname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(q, "SELECT tableoid, oid, " - "rcname AS conname, " - "'CHECK (' || rcsrc || ')' AS consrc, " - "true AS conislocal, true AS convalidated " - "FROM pg_relcheck " - "WHERE rcrelid = '%u'::oid " - "ORDER BY rcname", - tbinfo->dobj.catId.oid); - } - else - { - /* no tableoid in 7.0 */ - appendPQExpBuffer(q, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_relcheck') AS tableoid, " - "oid, rcname AS conname, " - "'CHECK (' || rcsrc || ')' AS consrc, " - "true AS conislocal, true AS convalidated " - "FROM pg_relcheck " - "WHERE rcrelid = '%u'::oid " - "ORDER BY rcname", - tbinfo->dobj.catId.oid); - } + res = ExecuteSqlQuery(fout, q->data, PGRES_TUPLES_OK); numConstrs = PQntuples(res); @@ -8303,8 +7443,7 @@ getTSParsers(Archive *fout, int *numTSParsers) prsinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_prsname)); prsinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_prsnamespace)), - prsinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_prsnamespace))); prsinfo[i].prsstart = atooid(PQgetvalue(res, i, i_prsstart)); prsinfo[i].prstoken = atooid(PQgetvalue(res, i, i_prstoken)); prsinfo[i].prsend = atooid(PQgetvalue(res, i, i_prsend)); @@ -8390,8 +7529,7 @@ getTSDictionaries(Archive *fout, int *numTSDicts) dictinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_dictname)); dictinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_dictnamespace)), - dictinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_dictnamespace))); dictinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); dictinfo[i].dicttemplate = atooid(PQgetvalue(res, i, i_dicttemplate)); if (PQgetisnull(res, i, i_dictinitoption)) @@ -8474,8 +7612,7 @@ getTSTemplates(Archive *fout, int *numTSTemplates) tmplinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_tmplname)); tmplinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_tmplnamespace)), - tmplinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_tmplnamespace))); tmplinfo[i].tmplinit = atooid(PQgetvalue(res, i, i_tmplinit)); tmplinfo[i].tmpllexize = atooid(PQgetvalue(res, i, i_tmpllexize)); @@ -8555,8 +7692,7 @@ getTSConfigurations(Archive *fout, int *numTSConfigs) cfginfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_cfgname)); cfginfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_cfgnamespace)), - cfginfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_cfgnamespace))); cfginfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); cfginfo[i].cfgparser = atooid(PQgetvalue(res, i, i_cfgparser)); @@ -8964,8 +8100,7 @@ getDefaultACLs(Archive *fout, int *numDefaultACLs) daclinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_defaclobjtype)); if (nspid != InvalidOid) - daclinfo[i].dobj.namespace = findNamespace(fout, nspid, - daclinfo[i].dobj.catId.oid); + daclinfo[i].dobj.namespace = findNamespace(fout, nspid); else daclinfo[i].dobj.namespace = NULL; @@ -9173,14 +8308,6 @@ findComments(Archive *fout, Oid classoid, Oid objoid, if (ncomments < 0) ncomments = collectComments(fout, &comments); - /* - * Pre-7.2, pg_description does not contain classoid, so collectComments - * just stores a zero. If there's a collision on object OID, well, you - * get duplicate comments. - */ - if (fout->remoteVersion < 70200) - classoid = 0; - /* * Do binary search to find some item matching the object. */ @@ -9268,25 +8395,9 @@ collectComments(Archive *fout, CommentItem **items) query = createPQExpBuffer(); - if (fout->remoteVersion >= 70300) - { - appendPQExpBufferStr(query, "SELECT description, classoid, objoid, objsubid " - "FROM pg_catalog.pg_description " - "ORDER BY classoid, objoid, objsubid"); - } - else if (fout->remoteVersion >= 70200) - { - appendPQExpBufferStr(query, "SELECT description, classoid, objoid, objsubid " - "FROM pg_description " - "ORDER BY classoid, objoid, objsubid"); - } - else - { - /* Note: this will fail to find attribute comments in pre-7.2... */ - appendPQExpBufferStr(query, "SELECT description, 0 AS classoid, objoid, 0 AS objsubid " - "FROM pg_description " - "ORDER BY objoid"); - } + appendPQExpBufferStr(query, "SELECT description, classoid, objoid, objsubid " + "FROM pg_catalog.pg_description " + "ORDER BY classoid, objoid, objsubid"); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -9468,10 +8579,6 @@ dumpNamespace(Archive *fout, NamespaceInfo *nspinfo) if (!nspinfo->dobj.dump || dopt->dataOnly) return; - /* don't dump dummy namespace from pre-7.3 source */ - if (strlen(nspinfo->dobj.name) == 0) - return; - q = createPQExpBuffer(); delq = createPQExpBuffer(); labelq = createPQExpBuffer(); @@ -10089,71 +9196,35 @@ dumpBaseType(Archive *fout, TypeInfo *tyinfo) "WHERE oid = '%u'::pg_catalog.oid", tyinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 80400) - { - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, typreceive, typsend, " - "typmodin, typmodout, typanalyze, " - "typreceive::pg_catalog.oid AS typreceiveoid, " - "typsend::pg_catalog.oid AS typsendoid, " - "typmodin::pg_catalog.oid AS typmodinoid, " - "typmodout::pg_catalog.oid AS typmodoutoid, " - "typanalyze::pg_catalog.oid AS typanalyzeoid, " - "typcategory, typispreferred, " - "typdelim, typbyval, typalign, typstorage, " - "false AS typcollatable, " - "pg_catalog.pg_get_expr(typdefaultbin, 0) AS typdefaultbin, typdefault " - "FROM pg_catalog.pg_type " - "WHERE oid = '%u'::pg_catalog.oid", - tyinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 80300) - { - /* Before 8.4, pg_get_expr does not allow 0 for its second arg */ - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, typreceive, typsend, " - "typmodin, typmodout, typanalyze, " - "typreceive::pg_catalog.oid AS typreceiveoid, " - "typsend::pg_catalog.oid AS typsendoid, " - "typmodin::pg_catalog.oid AS typmodinoid, " - "typmodout::pg_catalog.oid AS typmodoutoid, " - "typanalyze::pg_catalog.oid AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " - "typdelim, typbyval, typalign, typstorage, " - "false AS typcollatable, " - "pg_catalog.pg_get_expr(typdefaultbin, 'pg_catalog.pg_type'::pg_catalog.regclass) AS typdefaultbin, typdefault " - "FROM pg_catalog.pg_type " - "WHERE oid = '%u'::pg_catalog.oid", - tyinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 80000) + else if (fout->remoteVersion >= 80400) { appendPQExpBuffer(query, "SELECT typlen, " "typinput, typoutput, typreceive, typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "typanalyze, " + "typmodin, typmodout, typanalyze, " "typreceive::pg_catalog.oid AS typreceiveoid, " "typsend::pg_catalog.oid AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " + "typmodin::pg_catalog.oid AS typmodinoid, " + "typmodout::pg_catalog.oid AS typmodoutoid, " "typanalyze::pg_catalog.oid AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " + "typcategory, typispreferred, " "typdelim, typbyval, typalign, typstorage, " "false AS typcollatable, " - "pg_catalog.pg_get_expr(typdefaultbin, 'pg_catalog.pg_type'::pg_catalog.regclass) AS typdefaultbin, typdefault " + "pg_catalog.pg_get_expr(typdefaultbin, 0) AS typdefaultbin, typdefault " "FROM pg_catalog.pg_type " "WHERE oid = '%u'::pg_catalog.oid", tyinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70400) + else if (fout->remoteVersion >= 80300) { + /* Before 8.4, pg_get_expr does not allow 0 for its second arg */ appendPQExpBuffer(query, "SELECT typlen, " "typinput, typoutput, typreceive, typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " + "typmodin, typmodout, typanalyze, " "typreceive::pg_catalog.oid AS typreceiveoid, " "typsend::pg_catalog.oid AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " + "typmodin::pg_catalog.oid AS typmodinoid, " + "typmodout::pg_catalog.oid AS typmodoutoid, " + "typanalyze::pg_catalog.oid AS typanalyzeoid, " "'U' AS typcategory, false AS typispreferred, " "typdelim, typbyval, typalign, typstorage, " "false AS typcollatable, " @@ -10162,16 +9233,16 @@ dumpBaseType(Archive *fout, TypeInfo *tyinfo) "WHERE oid = '%u'::pg_catalog.oid", tyinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, " - "'-' AS typreceive, '-' AS typsend, " + "typinput, typoutput, typreceive, typsend, " "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " - "0 AS typreceiveoid, 0 AS typsendoid, " + "typanalyze, " + "typreceive::pg_catalog.oid AS typreceiveoid, " + "typsend::pg_catalog.oid AS typsendoid, " "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " + "typanalyze::pg_catalog.oid AS typanalyzeoid, " "'U' AS typcategory, false AS typispreferred, " "typdelim, typbyval, typalign, typstorage, " "false AS typcollatable, " @@ -10180,69 +9251,6 @@ dumpBaseType(Archive *fout, TypeInfo *tyinfo) "WHERE oid = '%u'::pg_catalog.oid", tyinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70200) - { - /* - * Note: although pre-7.3 catalogs contain typreceive and typsend, - * ignore them because they are not right. - */ - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, " - "'-' AS typreceive, '-' AS typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " - "0 AS typreceiveoid, 0 AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " - "typdelim, typbyval, typalign, typstorage, " - "false AS typcollatable, " - "NULL AS typdefaultbin, typdefault " - "FROM pg_type " - "WHERE oid = '%u'::oid", - tyinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - /* - * Ignore pre-7.2 typdefault; the field exists but has an unusable - * representation. - */ - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, " - "'-' AS typreceive, '-' AS typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " - "0 AS typreceiveoid, 0 AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " - "typdelim, typbyval, typalign, typstorage, " - "false AS typcollatable, " - "NULL AS typdefaultbin, NULL AS typdefault " - "FROM pg_type " - "WHERE oid = '%u'::oid", - tyinfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, " - "'-' AS typreceive, '-' AS typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " - "0 AS typreceiveoid, 0 AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " - "typdelim, typbyval, typalign, " - "'p'::char AS typstorage, " - "false AS typcollatable, " - "NULL AS typdefaultbin, NULL AS typdefault " - "FROM pg_type " - "WHERE oid = '%u'::oid", - tyinfo->dobj.catId.oid); - } res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -10300,30 +9308,19 @@ dumpBaseType(Archive *fout, TypeInfo *tyinfo) qtypname, (strcmp(typlen, "-1") == 0) ? "variable" : typlen); - if (fout->remoteVersion >= 70300) - { - /* regproc result is correctly quoted as of 7.3 */ - appendPQExpBuffer(q, ",\n INPUT = %s", typinput); - appendPQExpBuffer(q, ",\n OUTPUT = %s", typoutput); - if (OidIsValid(typreceiveoid)) - appendPQExpBuffer(q, ",\n RECEIVE = %s", typreceive); - if (OidIsValid(typsendoid)) - appendPQExpBuffer(q, ",\n SEND = %s", typsend); - if (OidIsValid(typmodinoid)) - appendPQExpBuffer(q, ",\n TYPMOD_IN = %s", typmodin); - if (OidIsValid(typmodoutoid)) - appendPQExpBuffer(q, ",\n TYPMOD_OUT = %s", typmodout); - if (OidIsValid(typanalyzeoid)) - appendPQExpBuffer(q, ",\n ANALYZE = %s", typanalyze); - } - else - { - /* regproc delivers an unquoted name before 7.3 */ - /* cannot combine these because fmtId uses static result area */ - appendPQExpBuffer(q, ",\n INPUT = %s", fmtId(typinput)); - appendPQExpBuffer(q, ",\n OUTPUT = %s", fmtId(typoutput)); - /* receive/send/typmodin/typmodout/analyze need not be printed */ - } + /* regproc result is sufficiently quoted already */ + appendPQExpBuffer(q, ",\n INPUT = %s", typinput); + appendPQExpBuffer(q, ",\n OUTPUT = %s", typoutput); + if (OidIsValid(typreceiveoid)) + appendPQExpBuffer(q, ",\n RECEIVE = %s", typreceive); + if (OidIsValid(typsendoid)) + appendPQExpBuffer(q, ",\n SEND = %s", typsend); + if (OidIsValid(typmodinoid)) + appendPQExpBuffer(q, ",\n TYPMOD_IN = %s", typmodin); + if (OidIsValid(typmodoutoid)) + appendPQExpBuffer(q, ",\n TYPMOD_OUT = %s", typmodout); + if (OidIsValid(typanalyzeoid)) + appendPQExpBuffer(q, ",\n ANALYZE = %s", typanalyze); if (strcmp(typcollatable, "t") == 0) appendPQExpBufferStr(q, ",\n COLLATABLE = true"); @@ -10468,7 +9465,6 @@ dumpDomain(Archive *fout, TypeInfo *tyinfo) } else { - /* We assume here that remoteVersion must be at least 70300 */ appendPQExpBuffer(query, "SELECT typnotnull, " "pg_catalog.format_type(typbasetype, typtypmod) AS typdefn, " "pg_catalog.pg_get_expr(typdefaultbin, 'pg_catalog.pg_type'::pg_catalog.regclass) AS typdefaultbin, " @@ -10669,9 +9665,8 @@ dumpCompositeType(Archive *fout, TypeInfo *tyinfo) else { /* - * We assume here that remoteVersion must be at least 70300. Since - * ALTER TYPE could not drop columns until 9.1, attisdropped should - * always be false. + * Since ALTER TYPE could not drop columns until 9.1, attisdropped + * should always be false. */ appendPQExpBuffer(query, "SELECT a.attname, " "pg_catalog.format_type(a.atttypid, a.atttypmod) AS atttypdefn, " @@ -10858,7 +9853,6 @@ dumpCompositeTypeColComments(Archive *fout, TypeInfo *tyinfo) query = createPQExpBuffer(); - /* We assume here that remoteVersion must be at least 70300 */ appendPQExpBuffer(query, "SELECT c.tableoid, a.attname, a.attnum " "FROM pg_catalog.pg_class c, pg_catalog.pg_attribute a " @@ -11442,7 +10436,7 @@ dumpFunc(Archive *fout, FuncInfo *finfo) "WHERE oid = '%u'::pg_catalog.oid", finfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 80000) + else { appendPQExpBuffer(query, "SELECT proretset, prosrc, probin, " @@ -11458,58 +10452,6 @@ dumpFunc(Archive *fout, FuncInfo *finfo) "WHERE oid = '%u'::pg_catalog.oid", finfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, - "SELECT proretset, prosrc, probin, " - "null AS proallargtypes, " - "null AS proargmodes, " - "null AS proargnames, " - "false AS proiswindow, " - "provolatile, proisstrict, prosecdef, " - "false AS proleakproof, " - "null AS proconfig, 0 AS procost, 0 AS prorows, " - "(SELECT lanname FROM pg_catalog.pg_language WHERE oid = prolang) AS lanname " - "FROM pg_catalog.pg_proc " - "WHERE oid = '%u'::pg_catalog.oid", - finfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, - "SELECT proretset, prosrc, probin, " - "null AS proallargtypes, " - "null AS proargmodes, " - "null AS proargnames, " - "false AS proiswindow, " - "case when proiscachable then 'i' else 'v' end AS provolatile, " - "proisstrict, " - "false AS prosecdef, " - "false AS proleakproof, " - "null AS proconfig, 0 AS procost, 0 AS prorows, " - "(SELECT lanname FROM pg_language WHERE oid = prolang) AS lanname " - "FROM pg_proc " - "WHERE oid = '%u'::oid", - finfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, - "SELECT proretset, prosrc, probin, " - "null AS proallargtypes, " - "null AS proargmodes, " - "null AS proargnames, " - "false AS proiswindow, " - "CASE WHEN proiscachable THEN 'i' ELSE 'v' END AS provolatile, " - "false AS proisstrict, " - "false AS prosecdef, " - "false AS proleakproof, " - "NULL AS proconfig, 0 AS procost, 0 AS prorows, " - "(SELECT lanname FROM pg_language WHERE oid = prolang) AS lanname " - "FROM pg_proc " - "WHERE oid = '%u'::oid", - finfo->dobj.catId.oid); - } res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -12082,7 +11024,6 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) PQExpBuffer labelq; PQExpBuffer oprid; PQExpBuffer details; - const char *name; PGresult *res; int i_oprkind; int i_oprcode; @@ -12143,7 +11084,7 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) "WHERE oid = '%u'::pg_catalog.oid", oprinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT oprkind, " "oprcode::pg_catalog.regprocedure, " @@ -12159,34 +11100,6 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) "WHERE oid = '%u'::pg_catalog.oid", oprinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT oprkind, oprcode, " - "CASE WHEN oprleft = 0 THEN '-' " - "ELSE format_type(oprleft, NULL) END AS oprleft, " - "CASE WHEN oprright = 0 THEN '-' " - "ELSE format_type(oprright, NULL) END AS oprright, " - "oprcom, oprnegate, oprrest, oprjoin, " - "(oprlsortop != 0) AS oprcanmerge, " - "oprcanhash " - "FROM pg_operator " - "WHERE oid = '%u'::oid", - oprinfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, "SELECT oprkind, oprcode, " - "CASE WHEN oprleft = 0 THEN '-'::name " - "ELSE (SELECT typname FROM pg_type WHERE oid = oprleft) END AS oprleft, " - "CASE WHEN oprright = 0 THEN '-'::name " - "ELSE (SELECT typname FROM pg_type WHERE oid = oprright) END AS oprright, " - "oprcom, oprnegate, oprrest, oprjoin, " - "(oprlsortop != 0) AS oprcanmerge, " - "oprcanhash " - "FROM pg_operator " - "WHERE oid = '%u'::oid", - oprinfo->dobj.catId.oid); - } res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -12229,12 +11142,8 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) if (strcmp(oprkind, "r") == 0 || strcmp(oprkind, "b") == 0) { - if (fout->remoteVersion >= 70100) - name = oprleft; - else - name = fmtId(oprleft); - appendPQExpBuffer(details, ",\n LEFTARG = %s", name); - appendPQExpBufferStr(oprid, name); + appendPQExpBuffer(details, ",\n LEFTARG = %s", oprleft); + appendPQExpBufferStr(oprid, oprleft); } else appendPQExpBufferStr(oprid, "NONE"); @@ -12242,12 +11151,8 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) if (strcmp(oprkind, "l") == 0 || strcmp(oprkind, "b") == 0) { - if (fout->remoteVersion >= 70100) - name = oprright; - else - name = fmtId(oprright); - appendPQExpBuffer(details, ",\n RIGHTARG = %s", name); - appendPQExpBuffer(oprid, ", %s)", name); + appendPQExpBuffer(details, ",\n RIGHTARG = %s", oprright); + appendPQExpBuffer(oprid, ", %s)", oprright); } else appendPQExpBufferStr(oprid, ", NONE)"); @@ -12334,40 +11239,34 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) * Returns allocated string of what to print, or NULL if function references * is InvalidOid. Returned string is expected to be free'd by the caller. * - * In 7.3 the input is a REGPROCEDURE display; we have to strip the - * argument-types part. In prior versions, the input is a REGPROC display. + * The input is a REGPROCEDURE display; we have to strip the argument-types + * part. */ static char * convertRegProcReference(Archive *fout, const char *proc) { + char *name; + char *paren; + bool inquote; + /* In all cases "-" means a null reference */ if (strcmp(proc, "-") == 0) return NULL; - if (fout->remoteVersion >= 70300) + name = pg_strdup(proc); + /* find non-double-quoted left paren */ + inquote = false; + for (paren = name; *paren; paren++) { - char *name; - char *paren; - bool inquote; - - name = pg_strdup(proc); - /* find non-double-quoted left paren */ - inquote = false; - for (paren = name; *paren; paren++) + if (*paren == '(' && !inquote) { - if (*paren == '(' && !inquote) - { - *paren = '\0'; - break; - } - if (*paren == '"') - inquote = !inquote; + *paren = '\0'; + break; } - return name; + if (*paren == '"') + inquote = !inquote; } - - /* REGPROC before 7.3 does not quote its result */ - return pg_strdup(fmtId(proc)); + return name; } /* @@ -12376,60 +11275,44 @@ convertRegProcReference(Archive *fout, const char *proc) * Returns an allocated string of what to print, or NULL to print nothing. * Caller is responsible for free'ing result string. * - * In 7.3 and up the input is a REGOPERATOR display; we have to strip the - * argument-types part, and add OPERATOR() decoration if the name is - * schema-qualified. In older versions, the input is just a numeric OID, - * which we search our operator list for. + * The input is a REGOPERATOR display; we have to strip the argument-types + * part, and add OPERATOR() decoration if the name is schema-qualified. */ static char * convertOperatorReference(Archive *fout, const char *opr) { - OprInfo *oprInfo; + char *name; + char *oname; + char *ptr; + bool inquote; + bool sawdot; /* In all cases "0" means a null reference */ if (strcmp(opr, "0") == 0) return NULL; - if (fout->remoteVersion >= 70300) - { - char *name; - char *oname; - char *ptr; - bool inquote; - bool sawdot; - - name = pg_strdup(opr); - /* find non-double-quoted left paren, and check for non-quoted dot */ - inquote = false; - sawdot = false; - for (ptr = name; *ptr; ptr++) + name = pg_strdup(opr); + /* find non-double-quoted left paren, and check for non-quoted dot */ + inquote = false; + sawdot = false; + for (ptr = name; *ptr; ptr++) + { + if (*ptr == '"') + inquote = !inquote; + else if (*ptr == '.' && !inquote) + sawdot = true; + else if (*ptr == '(' && !inquote) { - if (*ptr == '"') - inquote = !inquote; - else if (*ptr == '.' && !inquote) - sawdot = true; - else if (*ptr == '(' && !inquote) - { - *ptr = '\0'; - break; - } + *ptr = '\0'; + break; } - /* If not schema-qualified, don't need to add OPERATOR() */ - if (!sawdot) - return name; - oname = psprintf("OPERATOR(%s)", name); - free(name); - return oname; - } - - oprInfo = findOprByOid(atooid(opr)); - if (oprInfo == NULL) - { - write_msg(NULL, "WARNING: could not find operator with OID %s\n", - opr); - return NULL; } - return pg_strdup(oprInfo->dobj.name); + /* If not schema-qualified, don't need to add OPERATOR() */ + if (!sawdot) + return name; + oname = psprintf("OPERATOR(%s)", name); + free(name); + return oname; } /* @@ -12586,14 +11469,6 @@ dumpOpclass(Archive *fout, OpclassInfo *opcinfo) if (!opcinfo->dobj.dump || dopt->dataOnly) return; - /* - * XXX currently we do not implement dumping of operator classes from - * pre-7.3 databases. This could be done but it seems not worth the - * trouble. - */ - if (fout->remoteVersion < 70300) - return; - query = createPQExpBuffer(); q = createPQExpBuffer(); delq = createPQExpBuffer(); @@ -13343,7 +12218,7 @@ dumpConversion(Archive *fout, ConvInfo *convinfo) appendStringLiteralAH(q, conforencoding, fout); appendPQExpBufferStr(q, " TO "); appendStringLiteralAH(q, contoencoding, fout); - /* regproc is automatically quoted in 7.3 and above */ + /* regproc output is already sufficiently quoted */ appendPQExpBuffer(q, " FROM %s;\n", conproc); appendPQExpBuffer(labelq, "CONVERSION %s", fmtId(convinfo->dobj.name)); @@ -13569,7 +12444,7 @@ dumpAgg(Archive *fout, AggInfo *agginfo) "AND p.oid = '%u'::pg_catalog.oid", agginfo->aggfn.dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT aggtransfn, " "aggfinalfn, aggtranstype::pg_catalog.regtype, " @@ -13587,41 +12462,6 @@ dumpAgg(Archive *fout, AggInfo *agginfo) "AND p.oid = '%u'::pg_catalog.oid", agginfo->aggfn.dobj.catId.oid); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT aggtransfn, aggfinalfn, " - "format_type(aggtranstype, NULL) AS aggtranstype, " - "'-' AS aggcombinefn, '-' AS aggserialfn, " - "'-' AS aggdeserialfn, '-' AS aggmtransfn, " - "'-' AS aggminvtransfn, '-' AS aggmfinalfn, " - "0 AS aggmtranstype, false AS aggfinalextra, " - "false AS aggmfinalextra, 0 AS aggsortop, " - "false AS hypothetical, " - "0 AS aggtransspace, agginitval, " - "0 AS aggmtransspace, NULL AS aggminitval, " - "true AS convertok " - "FROM pg_aggregate " - "WHERE oid = '%u'::oid", - agginfo->aggfn.dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, "SELECT aggtransfn1 AS aggtransfn, " - "aggfinalfn, " - "(SELECT typname FROM pg_type WHERE oid = aggtranstype1) AS aggtranstype, " - "'-' AS aggcombinefn, '-' AS aggserialfn, " - "'-' AS aggdeserialfn, '-' AS aggmtransfn, " - "'-' AS aggminvtransfn, '-' AS aggmfinalfn, " - "0 AS aggmtranstype, false AS aggfinalextra, " - "false AS aggmfinalextra, 0 AS aggsortop, " - "false AS hypothetical, " - "0 AS aggtransspace, agginitval1 AS agginitval, " - "0 AS aggmtransspace, NULL AS aggminitval, " - "(aggtransfn2 = 0 and aggtranstype2 = 0 and agginitval2 is null) AS convertok " - "FROM pg_aggregate " - "WHERE oid = '%u'::oid", - agginfo->aggfn.dobj.catId.oid); - } res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -13701,28 +12541,9 @@ dumpAgg(Archive *fout, AggInfo *agginfo) return; } - if (fout->remoteVersion >= 70300) - { - /* If using 7.3's regproc or regtype, data is already quoted */ - appendPQExpBuffer(details, " SFUNC = %s,\n STYPE = %s", - aggtransfn, - aggtranstype); - } - else if (fout->remoteVersion >= 70100) - { - /* format_type quotes, regproc does not */ - appendPQExpBuffer(details, " SFUNC = %s,\n STYPE = %s", - fmtId(aggtransfn), - aggtranstype); - } - else - { - /* need quotes all around */ - appendPQExpBuffer(details, " SFUNC = %s,\n", - fmtId(aggtransfn)); - appendPQExpBuffer(details, " STYPE = %s", - fmtId(aggtranstype)); - } + /* regproc and regtype output is already sufficiently quoted */ + appendPQExpBuffer(details, " SFUNC = %s,\n STYPE = %s", + aggtransfn, aggtranstype); if (strcmp(aggtransspace, "0") != 0) { @@ -15121,19 +13942,9 @@ createViewAsClause(Archive *fout, TableInfo *tbinfo) int len; /* Fetch the view definition */ - if (fout->remoteVersion >= 70300) - { - /* Beginning in 7.3, viewname is not unique; rely on OID */ - appendPQExpBuffer(query, + appendPQExpBuffer(query, "SELECT pg_catalog.pg_get_viewdef('%u'::pg_catalog.oid) AS viewdef", - tbinfo->dobj.catId.oid); - } - else - { - appendPQExpBufferStr(query, "SELECT definition AS viewdef " - "FROM pg_views WHERE viewname = "); - appendStringLiteralAH(query, tbinfo->dobj.name, fout); - } + tbinfo->dobj.catId.oid); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -15370,17 +14181,10 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) { appendPQExpBufferStr(q, " WITH OPTIONS"); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(q, " %s", - tbinfo->atttypnames[j]); - } else { - /* If no format_type, fake it */ appendPQExpBuffer(q, " %s", - myFormatType(tbinfo->atttypnames[j], - tbinfo->atttypmod[j])); + tbinfo->atttypnames[j]); } /* Add collation if not default for the type */ @@ -16283,52 +15087,6 @@ dumpTableConstraintComment(Archive *fout, ConstraintInfo *coninfo) destroyPQExpBuffer(labelq); } -/* - * findLastBuiltInOid - - * find the last built in oid - * - * For 7.1 and 7.2, we do this by retrieving datlastsysoid from the - * pg_database entry for the current database - */ -static Oid -findLastBuiltinOid_V71(Archive *fout, const char *dbname) -{ - PGresult *res; - Oid last_oid; - PQExpBuffer query = createPQExpBuffer(); - - resetPQExpBuffer(query); - appendPQExpBufferStr(query, "SELECT datlastsysoid from pg_database where datname = "); - appendStringLiteralAH(query, dbname, fout); - - res = ExecuteSqlQueryForSingleRow(fout, query->data); - last_oid = atooid(PQgetvalue(res, 0, PQfnumber(res, "datlastsysoid"))); - PQclear(res); - destroyPQExpBuffer(query); - return last_oid; -} - -/* - * findLastBuiltInOid - - * find the last built in oid - * - * For 7.0, we do this by assuming that the last thing that initdb does is to - * create the pg_indexes view. This sucks in general, but seeing that 7.0.x - * initdb won't be changing anymore, it'll do. - */ -static Oid -findLastBuiltinOid_V70(Archive *fout) -{ - PGresult *res; - int last_oid; - - res = ExecuteSqlQueryForSingleRow(fout, - "SELECT oid FROM pg_class WHERE relname = 'pg_indexes'"); - last_oid = atooid(PQgetvalue(res, 0, PQfnumber(res, "oid"))); - PQclear(res); - return last_oid; -} - /* * dumpSequence * write the declaration (not data) of one user-defined sequence @@ -16703,13 +15461,9 @@ dumpTrigger(Archive *fout, TriggerInfo *tginfo) { if (OidIsValid(tginfo->tgconstrrelid)) { - /* If we are using regclass, name is already quoted */ - if (fout->remoteVersion >= 70300) - appendPQExpBuffer(query, " FROM %s\n ", - tginfo->tgconstrrelname); - else - appendPQExpBuffer(query, " FROM %s\n ", - fmtId(tginfo->tgconstrrelname)); + /* regclass output is already quoted */ + appendPQExpBuffer(query, " FROM %s\n ", + tginfo->tgconstrrelname); } if (!tginfo->tgdeferrable) appendPQExpBufferStr(query, "NOT "); @@ -16725,13 +15479,9 @@ dumpTrigger(Archive *fout, TriggerInfo *tginfo) else appendPQExpBufferStr(query, " FOR EACH STATEMENT\n "); - /* In 7.3, result of regproc is already quoted */ - if (fout->remoteVersion >= 70300) - appendPQExpBuffer(query, "EXECUTE PROCEDURE %s(", - tginfo->tgfname); - else - appendPQExpBuffer(query, "EXECUTE PROCEDURE %s(", - fmtId(tginfo->tgfname)); + /* regproc output is already sufficiently quoted */ + appendPQExpBuffer(query, "EXECUTE PROCEDURE %s(", + tginfo->tgfname); tgargs = (char *) PQunescapeBytea((unsigned char *) tginfo->tgargs, &lentgargs); @@ -16923,19 +15673,9 @@ dumpRule(Archive *fout, RuleInfo *rinfo) delcmd = createPQExpBuffer(); labelq = createPQExpBuffer(); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, - "SELECT pg_catalog.pg_get_ruledef('%u'::pg_catalog.oid) AS definition", - rinfo->dobj.catId.oid); - } - else - { - /* Rule name was unique before 7.3 ... */ - appendPQExpBuffer(query, - "SELECT pg_get_ruledef('%s') AS definition", - rinfo->dobj.name); - } + appendPQExpBuffer(query, + "SELECT pg_catalog.pg_get_ruledef('%u'::pg_catalog.oid) AS definition", + rinfo->dobj.catId.oid); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -17325,10 +16065,6 @@ getDependencies(Archive *fout) DumpableObject *dobj, *refdobj; - /* No dependency info available before 7.3 */ - if (fout->remoteVersion < 70300) - return; - if (g_verbose) write_msg(NULL, "reading dependency data\n"); @@ -17680,10 +16416,6 @@ selectSourceSchema(Archive *fout, const char *schemaName) /* This is checked by the callers already */ Assert(schemaName != NULL && *schemaName != '\0'); - /* Not relevant if fetching from pre-7.3 DB */ - if (fout->remoteVersion < 70300) - return; - query = createPQExpBuffer(); appendPQExpBuffer(query, "SET search_path = %s", fmtId(schemaName)); @@ -17699,8 +16431,8 @@ selectSourceSchema(Archive *fout, const char *schemaName) * getFormattedTypeName - retrieve a nicely-formatted type name for the * given type name. * - * NB: in 7.3 and up the result may depend on the currently-selected - * schema; this is why we don't try to cache the names. + * NB: the result may depend on the currently-selected search_path; this is + * why we don't try to cache the names. */ static char * getFormattedTypeName(Archive *fout, Oid oid, OidOptions opts) @@ -17722,36 +16454,13 @@ getFormattedTypeName(Archive *fout, Oid oid, OidOptions opts) } query = createPQExpBuffer(); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, "SELECT pg_catalog.format_type('%u'::pg_catalog.oid, NULL)", - oid); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT format_type('%u'::oid, NULL)", - oid); - } - else - { - appendPQExpBuffer(query, "SELECT typname " - "FROM pg_type " - "WHERE oid = '%u'::oid", - oid); - } + appendPQExpBuffer(query, "SELECT pg_catalog.format_type('%u'::pg_catalog.oid, NULL)", + oid); res = ExecuteSqlQueryForSingleRow(fout, query->data); - if (fout->remoteVersion >= 70100) - { - /* already quoted */ - result = pg_strdup(PQgetvalue(res, 0, 0)); - } - else - { - /* may need to quote it */ - result = pg_strdup(fmtId(PQgetvalue(res, 0, 0))); - } + /* result of format_type is already quoted */ + result = pg_strdup(PQgetvalue(res, 0, 0)); PQclear(res); destroyPQExpBuffer(query); @@ -17759,76 +16468,6 @@ getFormattedTypeName(Archive *fout, Oid oid, OidOptions opts) return result; } -/* - * myFormatType --- local implementation of format_type for use with 7.0. - */ -static char * -myFormatType(const char *typname, int32 typmod) -{ - char *result; - bool isarray = false; - PQExpBuffer buf = createPQExpBuffer(); - - /* Handle array types */ - if (typname[0] == '_') - { - isarray = true; - typname++; - } - - /* Show lengths on bpchar and varchar */ - if (strcmp(typname, "bpchar") == 0) - { - int len = (typmod - VARHDRSZ); - - appendPQExpBufferStr(buf, "character"); - if (len > 1) - appendPQExpBuffer(buf, "(%d)", - typmod - VARHDRSZ); - } - else if (strcmp(typname, "varchar") == 0) - { - appendPQExpBufferStr(buf, "character varying"); - if (typmod != -1) - appendPQExpBuffer(buf, "(%d)", - typmod - VARHDRSZ); - } - else if (strcmp(typname, "numeric") == 0) - { - appendPQExpBufferStr(buf, "numeric"); - if (typmod != -1) - { - int32 tmp_typmod; - int precision; - int scale; - - tmp_typmod = typmod - VARHDRSZ; - precision = (tmp_typmod >> 16) & 0xffff; - scale = tmp_typmod & 0xffff; - appendPQExpBuffer(buf, "(%d,%d)", - precision, scale); - } - } - - /* - * char is an internal single-byte data type; Let's make sure we force it - * through with quotes. - thomas 1998-12-13 - */ - else if (strcmp(typname, "char") == 0) - appendPQExpBufferStr(buf, "\"char\""); - else - appendPQExpBufferStr(buf, fmtId(typname)); - - /* Append array qualifier for array types */ - if (isarray) - appendPQExpBufferStr(buf, "[]"); - - result = pg_strdup(buf->data); - destroyPQExpBuffer(buf); - - return result; -} - /* * Return a column list clause for the given relation. * diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 2bfa2d9742..a60cf95733 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -605,7 +605,6 @@ extern void parseOidArray(const char *str, Oid *array, int arraysize); extern void sortDumpableObjects(DumpableObject **objs, int numObjs, DumpId preBoundaryId, DumpId postBoundaryId); extern void sortDumpableObjectsByTypeName(DumpableObject **objs, int numObjs); -extern void sortDumpableObjectsByTypeOid(DumpableObject **objs, int numObjs); extern void sortDataAndIndexObjectsBySize(DumpableObject **objs, int numObjs); /* diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c index d87f08d356..195b84a0d4 100644 --- a/src/bin/pg_dump/pg_dump_sort.c +++ b/src/bin/pg_dump/pg_dump_sort.c @@ -23,63 +23,7 @@ static const char *modulename = gettext_noop("sorter"); /* - * Sort priority for object types when dumping a pre-7.3 database. - * Objects are sorted by priority levels, and within an equal priority level - * by OID. (This is a relatively crude hack to provide semi-reasonable - * behavior for old databases without full dependency info.) Note: collations, - * extensions, text search, foreign-data, materialized view, event trigger, - * policies, transforms, access methods and default ACL objects can't really - * happen here, so the rather bogus priorities for them don't matter. - * - * NOTE: object-type priorities must match the section assignments made in - * pg_dump.c; that is, PRE_DATA objects must sort before DO_PRE_DATA_BOUNDARY, - * POST_DATA objects must sort after DO_POST_DATA_BOUNDARY, and DATA objects - * must sort between them. - */ -static const int oldObjectTypePriority[] = -{ - 1, /* DO_NAMESPACE */ - 1, /* DO_EXTENSION */ - 2, /* DO_TYPE */ - 2, /* DO_SHELL_TYPE */ - 2, /* DO_FUNC */ - 3, /* DO_AGG */ - 3, /* DO_OPERATOR */ - 3, /* DO_ACCESS_METHOD */ - 4, /* DO_OPCLASS */ - 4, /* DO_OPFAMILY */ - 4, /* DO_COLLATION */ - 5, /* DO_CONVERSION */ - 6, /* DO_TABLE */ - 8, /* DO_ATTRDEF */ - 15, /* DO_INDEX */ - 16, /* DO_RULE */ - 17, /* DO_TRIGGER */ - 14, /* DO_CONSTRAINT */ - 18, /* DO_FK_CONSTRAINT */ - 2, /* DO_PROCLANG */ - 2, /* DO_CAST */ - 11, /* DO_TABLE_DATA */ - 7, /* DO_DUMMY_TYPE */ - 4, /* DO_TSPARSER */ - 4, /* DO_TSDICT */ - 4, /* DO_TSTEMPLATE */ - 4, /* DO_TSCONFIG */ - 4, /* DO_FDW */ - 4, /* DO_FOREIGN_SERVER */ - 19, /* DO_DEFAULT_ACL */ - 4, /* DO_TRANSFORM */ - 9, /* DO_BLOB */ - 12, /* DO_BLOB_DATA */ - 10, /* DO_PRE_DATA_BOUNDARY */ - 13, /* DO_POST_DATA_BOUNDARY */ - 20, /* DO_EVENT_TRIGGER */ - 15, /* DO_REFRESH_MATVIEW */ - 21 /* DO_POLICY */ -}; - -/* - * Sort priority for object types when dumping newer databases. + * Sort priority for database object types. * Objects are sorted by type, and within a type by name. * * NOTE: object-type priorities must match the section assignments made in @@ -87,7 +31,7 @@ static const int oldObjectTypePriority[] = * POST_DATA objects must sort after DO_POST_DATA_BOUNDARY, and DATA objects * must sort between them. */ -static const int newObjectTypePriority[] = +static const int dbObjectTypePriority[] = { 1, /* DO_NAMESPACE */ 4, /* DO_EXTENSION */ @@ -134,7 +78,6 @@ static DumpId postDataBoundId; static int DOTypeNameCompare(const void *p1, const void *p2); -static int DOTypeOidCompare(const void *p1, const void *p2); static bool TopoSort(DumpableObject **objs, int numObjs, DumpableObject **ordering, @@ -266,8 +209,8 @@ DOTypeNameCompare(const void *p1, const void *p2) int cmpval; /* Sort by type */ - cmpval = newObjectTypePriority[obj1->objType] - - newObjectTypePriority[obj2->objType]; + cmpval = dbObjectTypePriority[obj1->objType] - + dbObjectTypePriority[obj2->objType]; if (cmpval != 0) return cmpval; @@ -345,37 +288,6 @@ DOTypeNameCompare(const void *p1, const void *p2) } -/* - * Sort the given objects into a type/OID-based ordering - * - * This is used with pre-7.3 source databases as a crude substitute for the - * lack of dependency information. - */ -void -sortDumpableObjectsByTypeOid(DumpableObject **objs, int numObjs) -{ - if (numObjs > 1) - qsort((void *) objs, numObjs, sizeof(DumpableObject *), - DOTypeOidCompare); -} - -static int -DOTypeOidCompare(const void *p1, const void *p2) -{ - DumpableObject *obj1 = *(DumpableObject *const *) p1; - DumpableObject *obj2 = *(DumpableObject *const *) p2; - int cmpval; - - cmpval = oldObjectTypePriority[obj1->objType] - - oldObjectTypePriority[obj2->objType]; - - if (cmpval != 0) - return cmpval; - - return oidcmp(obj1->catId.oid, obj2->catId.oid); -} - - /* * Sort the given objects into a safe dump order using dependency * information (to the extent we have it available). diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index b5efb46019..82157e5620 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -480,10 +480,7 @@ main(int argc, char *argv[]) dropDBs(conn); if (!roles_only && !no_tablespaces) - { - if (server_version >= 80000) - dropTablespaces(conn); - } + dropTablespaces(conn); if (!tablespaces_only) dropRoles(conn); @@ -505,12 +502,9 @@ main(int argc, char *argv[]) dumpGroups(conn); } + /* Dump tablespaces */ if (!roles_only && !no_tablespaces) - { - /* Dump tablespaces */ - if (server_version >= 80000) - dumpTablespaces(conn); - } + dumpTablespaces(conn); /* Dump CREATE DATABASE commands */ if (binary_upgrade || (!globals_only && !roles_only && !tablespaces_only)) @@ -886,9 +880,8 @@ dumpRoles(PGconn *conn) * We do it this way because config settings for roles could mention the * names of other roles. */ - if (server_version >= 70300) - for (i = 0; i < PQntuples(res); i++) - dumpUserConfig(conn, PQgetvalue(res, i, i_rolname)); + for (i = 0; i < PQntuples(res); i++) + dumpUserConfig(conn, PQgetvalue(res, i, i_rolname)); PQclear(res); @@ -1204,16 +1197,10 @@ dropDBs(PGconn *conn) PGresult *res; int i; - if (server_version >= 70100) - res = executeQuery(conn, - "SELECT datname " - "FROM pg_database d " - "WHERE datallowconn ORDER BY 1"); - else - res = executeQuery(conn, - "SELECT datname " - "FROM pg_database d " - "ORDER BY 1"); + res = executeQuery(conn, + "SELECT datname " + "FROM pg_database d " + "WHERE datallowconn ORDER BY 1"); if (PQntuples(res) > 0) fprintf(OPF, "--\n-- Drop databases\n--\n\n"); @@ -1269,12 +1256,10 @@ dumpCreateDB(PGconn *conn) * We will dump encoding and locale specifications in the CREATE DATABASE * commands for just those databases with values different from defaults. * - * We consider template0's encoding and locale (or, pre-7.1, template1's) - * to define the installation default. Pre-8.4 installations do not have - * per-database locale settings; for them, every database must necessarily - * be using the installation default, so there's no need to do anything - * (which is good, since in very old versions there is no good way to find - * out what the installation locale is anyway...) + * We consider template0's encoding and locale to define the installation + * default. Pre-8.4 installations do not have per-database locale + * settings; for them, every database must necessarily be using the + * installation default, so there's no need to do anything. */ if (server_version >= 80400) res = executeQuery(conn, @@ -1282,18 +1267,12 @@ dumpCreateDB(PGconn *conn) "datcollate, datctype " "FROM pg_database " "WHERE datname = 'template0'"); - else if (server_version >= 70100) - res = executeQuery(conn, - "SELECT pg_encoding_to_char(encoding), " - "null::text AS datcollate, null::text AS datctype " - "FROM pg_database " - "WHERE datname = 'template0'"); else res = executeQuery(conn, "SELECT pg_encoding_to_char(encoding), " "null::text AS datcollate, null::text AS datctype " "FROM pg_database " - "WHERE datname = 'template1'"); + "WHERE datname = 'template0'"); /* If for some reason the template DB isn't there, treat as unknown */ if (PQntuples(res) > 0) @@ -1371,7 +1350,7 @@ dumpCreateDB(PGconn *conn) "(SELECT spcname FROM pg_tablespace t WHERE t.oid = d.dattablespace) AS dattablespace " "FROM pg_database d LEFT JOIN pg_authid u ON (datdba = u.oid) " "WHERE datallowconn ORDER BY 1"); - else if (server_version >= 80000) + else res = executeQuery(conn, "SELECT datname, " "coalesce(usename, (select usename from pg_shadow where usesysid=(select datdba from pg_database where datname='template0'))), " @@ -1382,47 +1361,6 @@ dumpCreateDB(PGconn *conn) "(SELECT spcname FROM pg_tablespace t WHERE t.oid = d.dattablespace) AS dattablespace " "FROM pg_database d LEFT JOIN pg_shadow u ON (datdba = usesysid) " "WHERE datallowconn ORDER BY 1"); - else if (server_version >= 70300) - res = executeQuery(conn, - "SELECT datname, " - "coalesce(usename, (select usename from pg_shadow where usesysid=(select datdba from pg_database where datname='template0'))), " - "pg_encoding_to_char(d.encoding), " - "null::text AS datcollate, null::text AS datctype, datfrozenxid, 0 AS datminmxid, " - "datistemplate, datacl, '' as rdatacl, " - "-1 as datconnlimit, " - "'pg_default' AS dattablespace " - "FROM pg_database d LEFT JOIN pg_shadow u ON (datdba = usesysid) " - "WHERE datallowconn ORDER BY 1"); - else if (server_version >= 70100) - res = executeQuery(conn, - "SELECT datname, " - "coalesce(" - "(select usename from pg_shadow where usesysid=datdba), " - "(select usename from pg_shadow where usesysid=(select datdba from pg_database where datname='template0'))), " - "pg_encoding_to_char(d.encoding), " - "null::text AS datcollate, null::text AS datctype, 0 AS datfrozenxid, 0 AS datminmxid, " - "datistemplate, '' as datacl, '' as rdatacl, " - "-1 as datconnlimit, " - "'pg_default' AS dattablespace " - "FROM pg_database d " - "WHERE datallowconn ORDER BY 1"); - else - { - /* - * Note: 7.0 fails to cope with sub-select in COALESCE, so just deal - * with getting a NULL by not printing any OWNER clause. - */ - res = executeQuery(conn, - "SELECT datname, " - "(select usename from pg_shadow where usesysid=datdba), " - "pg_encoding_to_char(d.encoding), " - "null::text AS datcollate, null::text AS datctype, 0 AS datfrozenxid, 0 AS datminmxid, " - "'f' as datistemplate, " - "'' as datacl, '' as rdatacl, -1 as datconnlimit, " - "'pg_default' AS dattablespace " - "FROM pg_database d " - "ORDER BY 1"); - } for (i = 0; i < PQntuples(res); i++) { @@ -1541,8 +1479,7 @@ dumpCreateDB(PGconn *conn) fprintf(OPF, "%s", buf->data); - if (server_version >= 70300) - dumpDatabaseConfig(conn, dbname); + dumpDatabaseConfig(conn, dbname); free(fdbname); } @@ -1738,10 +1675,7 @@ dumpDatabases(PGconn *conn) PGresult *res; int i; - if (server_version >= 70100) - res = executeQuery(conn, "SELECT datname FROM pg_database WHERE datallowconn ORDER BY 1"); - else - res = executeQuery(conn, "SELECT datname FROM pg_database ORDER BY 1"); + res = executeQuery(conn, "SELECT datname FROM pg_database WHERE datallowconn ORDER BY 1"); for (i = 0; i < PQntuples(res); i++) { @@ -2062,11 +1996,11 @@ connectDatabase(const char *dbname, const char *connection_string, my_version = PG_VERSION_NUM; /* - * We allow the server to be back to 7.0, and up to any minor release of + * We allow the server to be back to 8.0, and up to any minor release of * our own major version. (See also version check in pg_dump.c.) */ if (my_version != server_version - && (server_version < 70000 || + && (server_version < 80000 || (server_version / 100) > (my_version / 100))) { fprintf(stderr, _("server version: %s; %s version: %s\n"), @@ -2076,11 +2010,9 @@ connectDatabase(const char *dbname, const char *connection_string, } /* - * On 7.3 and later, make sure we are not fooled by non-system schemas in - * the search path. + * Make sure we are not fooled by non-system schemas in the search path. */ - if (server_version >= 70300) - executeCommand(conn, "SET search_path = pg_catalog"); + executeCommand(conn, "SET search_path = pg_catalog"); return conn; } From 8518583cdb10340bab3464121629a1a9ec387afa Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 12 Oct 2016 12:45:50 -0400 Subject: [PATCH 311/871] Provide DLLEXPORT markers for C functions via PG_FUNCTION_INFO_V1 macro. This isn't really necessary for our own code, because we use a .DEF file in MSVC builds (see gendef.pl), or --export-all-symbols in MinGW and Cygwin builds, to ensure that all global symbols in loadable modules will be exported on Windows. However, third-party authors might use different build processes that need this marker, and it's harmless enough for our own builds. To some extent, this is an oversight in commit e7128e8db, so back-patch to 9.4 where that was added. Laurenz Albe Discussion: --- src/include/fmgr.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/include/fmgr.h b/src/include/fmgr.h index 0491e2e1d1..18c224ecfa 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -344,11 +344,17 @@ typedef const Pg_finfo_record *(*PGFInfoFunction) (void); /* * Macro to build an info function associated with the given function name. - * Win32 loadable functions usually link with 'dlltool --export-all', but it - * doesn't hurt to add PGDLLIMPORT in case they don't. + * + * As a convenience, also provide an "extern" declaration for the given + * function name, so that writers of C functions need not write that too. + * + * On Windows, the function and info function must be exported. Our normal + * build processes take care of that via .DEF files or --export-all-symbols. + * We add PGDLLEXPORT nonetheless so that C functions built with a + * different build process are guaranteed to be exported. */ #define PG_FUNCTION_INFO_V1(funcname) \ -Datum funcname(PG_FUNCTION_ARGS); \ +extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); \ extern PGDLLEXPORT const Pg_finfo_record * CppConcat(pg_finfo_,funcname)(void); \ const Pg_finfo_record * \ CppConcat(pg_finfo_,funcname) (void) \ From 5c80642aa8de8393b08cd3cbf612b325cedd98dc Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 12 Oct 2016 14:54:08 -0400 Subject: [PATCH 312/871] Remove unnecessary int2vector-specific hash function and equality operator. These functions were originally added in commit d8cedf67a to support use of int2vector columns as catcache lookup keys. However, there are no catcaches that use such columns. (Indeed I now think it must always have been dead code: a catcache with such a key column would need an underlying unique index on the column, but we've never had an int2vector btree opclass.) Getting rid of the int2vector-specific operator and function does not lose any functionality, because operations on int2vectors will now fall back to the generic anyarray support. This avoids a wart that a btree index on an int2vector column (made using anyarray_ops) would fail to match equality searches, because int2vectoreq wasn't a member of the opclass. We don't really care much about that, since int2vector is not meant as a type for users to use, but it's silly to have extra code and less functionality. If we ever do want a catcache to be indexed by an int2vector column, we'd need to put back full btree and hash opclasses for int2vector, comparable to the support for oidvector. (The anyarray code can't be used at such a low level, because it needs to do catcache lookups.) But we'll deal with that if/when the need arises. Also worth noting is that removal of the hash int2vector_ops opclass will break any user-created hash indexes on int2vector columns. While hash anyarray_ops would serve the same purpose, it would probably not compute the same hash values and thus wouldn't be on-disk-compatible. Given that int2vector isn't a user-facing type and we're planning other incompatible changes in hash indexes for v10 anyway, this doesn't seem like something to worry about, but it's probably worth mentioning here. Amit Langote Discussion: --- src/backend/access/hash/hashfunc.c | 8 -------- src/backend/utils/adt/int.c | 15 --------------- src/backend/utils/cache/catcache.c | 5 ----- src/include/access/hash.h | 1 - src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_amop.h | 2 -- src/include/catalog/pg_amproc.h | 1 - src/include/catalog/pg_opclass.h | 1 - src/include/catalog/pg_operator.h | 2 -- src/include/catalog/pg_opfamily.h | 1 - src/include/catalog/pg_proc.h | 3 --- src/include/utils/builtins.h | 1 - 12 files changed, 1 insertion(+), 41 deletions(-) diff --git a/src/backend/access/hash/hashfunc.c b/src/backend/access/hash/hashfunc.c index 614f4ff2f5..12dce2e2b4 100644 --- a/src/backend/access/hash/hashfunc.c +++ b/src/backend/access/hash/hashfunc.c @@ -130,14 +130,6 @@ hashoidvector(PG_FUNCTION_ARGS) return hash_any((unsigned char *) key->values, key->dim1 * sizeof(Oid)); } -Datum -hashint2vector(PG_FUNCTION_ARGS) -{ - int2vector *key = (int2vector *) PG_GETARG_POINTER(0); - - return hash_any((unsigned char *) key->values, key->dim1 * sizeof(int16)); -} - Datum hashname(PG_FUNCTION_ARGS) { diff --git a/src/backend/utils/adt/int.c b/src/backend/utils/adt/int.c index 29d92a7d04..8c44014ab6 100644 --- a/src/backend/utils/adt/int.c +++ b/src/backend/utils/adt/int.c @@ -254,21 +254,6 @@ int2vectorsend(PG_FUNCTION_ARGS) return array_send(fcinfo); } -/* - * We don't have a complete set of int2vector support routines, - * but we need int2vectoreq for catcache indexing. - */ -Datum -int2vectoreq(PG_FUNCTION_ARGS) -{ - int2vector *a = (int2vector *) PG_GETARG_POINTER(0); - int2vector *b = (int2vector *) PG_GETARG_POINTER(1); - - if (a->dim1 != b->dim1) - PG_RETURN_BOOL(false); - PG_RETURN_BOOL(memcmp(a->values, b->values, a->dim1 * sizeof(int16)) == 0); -} - /***************************************************************************** * PUBLIC ROUTINES * diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c index db7099fc0e..6016d192a5 100644 --- a/src/backend/utils/cache/catcache.c +++ b/src/backend/utils/cache/catcache.c @@ -126,11 +126,6 @@ GetCCHashEqFuncs(Oid keytype, PGFunction *hashfunc, RegProcedure *eqfunc) *eqfunc = F_INT2EQ; break; - case INT2VECTOROID: - *hashfunc = hashint2vector; - - *eqfunc = F_INT2VECTOREQ; - break; case INT4OID: *hashfunc = hashint4; diff --git a/src/include/access/hash.h b/src/include/access/hash.h index 491d4c90bb..725e2f251c 100644 --- a/src/include/access/hash.h +++ b/src/include/access/hash.h @@ -283,7 +283,6 @@ extern Datum hashenum(PG_FUNCTION_ARGS); extern Datum hashfloat4(PG_FUNCTION_ARGS); extern Datum hashfloat8(PG_FUNCTION_ARGS); extern Datum hashoidvector(PG_FUNCTION_ARGS); -extern Datum hashint2vector(PG_FUNCTION_ARGS); extern Datum hashname(PG_FUNCTION_ARGS); extern Datum hashtext(PG_FUNCTION_ARGS); extern Datum hashvarlena(PG_FUNCTION_ARGS); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 3fdd0d6129..bfa6b870a1 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201609261 +#define CATALOG_VERSION_NO 201610121 #endif diff --git a/src/include/catalog/pg_amop.h b/src/include/catalog/pg_amop.h index 15b629029f..e4c3515032 100644 --- a/src/include/catalog/pg_amop.h +++ b/src/include/catalog/pg_amop.h @@ -573,8 +573,6 @@ DATA(insert ( 2040 1114 1114 1 s 2060 405 0 )); DATA(insert ( 2222 16 16 1 s 91 405 0 )); /* bytea_ops */ DATA(insert ( 2223 17 17 1 s 1955 405 0 )); -/* int2vector_ops */ -DATA(insert ( 2224 22 22 1 s 386 405 0 )); /* xid_ops */ DATA(insert ( 2225 28 28 1 s 352 405 0 )); /* cid_ops */ diff --git a/src/include/catalog/pg_amproc.h b/src/include/catalog/pg_amproc.h index 1b654d5be4..f01a5b4fb1 100644 --- a/src/include/catalog/pg_amproc.h +++ b/src/include/catalog/pg_amproc.h @@ -173,7 +173,6 @@ DATA(insert ( 2001 1266 1266 1 1696 )); DATA(insert ( 2040 1114 1114 1 2039 )); DATA(insert ( 2222 16 16 1 454 )); DATA(insert ( 2223 17 17 1 456 )); -DATA(insert ( 2224 22 22 1 398 )); DATA(insert ( 2225 28 28 1 450 )); DATA(insert ( 2226 29 29 1 450 )); DATA(insert ( 2227 702 702 1 450 )); diff --git a/src/include/catalog/pg_opclass.h b/src/include/catalog/pg_opclass.h index 5900cdc5b0..ade8da3f34 100644 --- a/src/include/catalog/pg_opclass.h +++ b/src/include/catalog/pg_opclass.h @@ -168,7 +168,6 @@ DATA(insert ( 403 bpchar_pattern_ops PGNSP PGUID 2097 1042 f 0 )); DATA(insert ( 403 money_ops PGNSP PGUID 2099 790 t 0 )); DATA(insert ( 405 bool_ops PGNSP PGUID 2222 16 t 0 )); DATA(insert ( 405 bytea_ops PGNSP PGUID 2223 17 t 0 )); -DATA(insert ( 405 int2vector_ops PGNSP PGUID 2224 22 t 0 )); DATA(insert ( 403 tid_ops PGNSP PGUID 2789 27 t 0 )); DATA(insert ( 405 xid_ops PGNSP PGUID 2225 28 t 0 )); DATA(insert ( 405 cid_ops PGNSP PGUID 2226 29 t 0 )); diff --git a/src/include/catalog/pg_operator.h b/src/include/catalog/pg_operator.h index b8f06b3500..26fa6188fa 100644 --- a/src/include/catalog/pg_operator.h +++ b/src/include/catalog/pg_operator.h @@ -156,8 +156,6 @@ DATA(insert OID = 389 ( "!!" PGNSP PGUID l f f 0 20 1700 0 0 numeric_fac DESCR("deprecated, use ! instead"); DATA(insert OID = 385 ( "=" PGNSP PGUID b f t 29 29 16 385 0 cideq eqsel eqjoinsel )); DESCR("equal"); -DATA(insert OID = 386 ( "=" PGNSP PGUID b f t 22 22 16 386 0 int2vectoreq eqsel eqjoinsel )); -DESCR("equal"); DATA(insert OID = 387 ( "=" PGNSP PGUID b t f 27 27 16 387 402 tideq eqsel eqjoinsel )); DESCR("equal"); diff --git a/src/include/catalog/pg_opfamily.h b/src/include/catalog/pg_opfamily.h index ac6b304787..7ba23e5cf9 100644 --- a/src/include/catalog/pg_opfamily.h +++ b/src/include/catalog/pg_opfamily.h @@ -117,7 +117,6 @@ DATA(insert OID = 2099 ( 403 money_ops PGNSP PGUID )); DATA(insert OID = 2222 ( 405 bool_ops PGNSP PGUID )); #define BOOL_HASH_FAM_OID 2222 DATA(insert OID = 2223 ( 405 bytea_ops PGNSP PGUID )); -DATA(insert OID = 2224 ( 405 int2vector_ops PGNSP PGUID )); DATA(insert OID = 2789 ( 403 tid_ops PGNSP PGUID )); DATA(insert OID = 2225 ( 405 xid_ops PGNSP PGUID )); DATA(insert OID = 2226 ( 405 cid_ops PGNSP PGUID )); diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index e2d08babda..17ec71d47e 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -540,7 +540,6 @@ DATA(insert OID = 313 ( int4 PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 DESCR("convert int2 to int4"); DATA(insert OID = 314 ( int2 PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 21 "23" _null_ _null_ _null_ _null_ _null_ i4toi2 _null_ _null_ _null_ )); DESCR("convert int4 to int2"); -DATA(insert OID = 315 ( int2vectoreq PGNSP PGUID 12 1 0 0 0 f f f f t f i s 2 0 16 "22 22" _null_ _null_ _null_ _null_ _null_ int2vectoreq _null_ _null_ _null_ )); DATA(insert OID = 316 ( float8 PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 701 "23" _null_ _null_ _null_ _null_ _null_ i4tod _null_ _null_ _null_ )); DESCR("convert int4 to float8"); DATA(insert OID = 317 ( int4 PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "701" _null_ _null_ _null_ _null_ _null_ dtoi4 _null_ _null_ _null_ )); @@ -687,8 +686,6 @@ DATA(insert OID = 457 ( hashoidvector PGNSP PGUID 12 1 0 0 0 f f f f t f i s DESCR("hash"); DATA(insert OID = 329 ( hash_aclitem PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "1033" _null_ _null_ _null_ _null_ _null_ hash_aclitem _null_ _null_ _null_ )); DESCR("hash"); -DATA(insert OID = 398 ( hashint2vector PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "22" _null_ _null_ _null_ _null_ _null_ hashint2vector _null_ _null_ _null_ )); -DESCR("hash"); DATA(insert OID = 399 ( hashmacaddr PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "829" _null_ _null_ _null_ _null_ _null_ hashmacaddr _null_ _null_ _null_ )); DESCR("hash"); DATA(insert OID = 422 ( hashinet PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "869" _null_ _null_ _null_ _null_ _null_ hashinet _null_ _null_ _null_ )); diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 2ae212a9c3..90f5132b03 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -192,7 +192,6 @@ extern Datum int2vectorin(PG_FUNCTION_ARGS); extern Datum int2vectorout(PG_FUNCTION_ARGS); extern Datum int2vectorrecv(PG_FUNCTION_ARGS); extern Datum int2vectorsend(PG_FUNCTION_ARGS); -extern Datum int2vectoreq(PG_FUNCTION_ARGS); extern Datum int4in(PG_FUNCTION_ARGS); extern Datum int4out(PG_FUNCTION_ARGS); extern Datum int4recv(PG_FUNCTION_ARGS); From c0a3b211bcb790a8d76022cb2b3ffe9795aaf5e9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 12 Oct 2016 15:11:31 -0400 Subject: [PATCH 313/871] pg_dump's getTypes() needn't retrieve typinput or typoutput anymore. Commit 64f3524e2 removed the stanza of code that examined these values. I failed to notice they were unnecessary because my compiler didn't warn about the un-read variables. Noted by Peter Eisentraut. --- src/bin/pg_dump/pg_dump.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index c821f3b0ee..28ec5bf383 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -3688,8 +3688,6 @@ getTypes(Archive *fout, int *numTypes) int i_inittypacl; int i_initrtypacl; int i_rolname; - int i_typinput; - int i_typoutput; int i_typelem; int i_typrelid; int i_typrelkind; @@ -3734,8 +3732,7 @@ getTypes(Archive *fout, int *numTypes) "%s AS inittypacl, " "%s AS initrtypacl, " "(%s t.typowner) AS rolname, " - "t.typinput::oid AS typinput, " - "t.typoutput::oid AS typoutput, t.typelem, t.typrelid, " + "t.typelem, t.typrelid, " "CASE WHEN t.typrelid = 0 THEN ' '::\"char\" " "ELSE (SELECT relkind FROM pg_class WHERE oid = t.typrelid) END AS typrelkind, " "t.typtype, t.typisdefined, " @@ -3763,8 +3760,7 @@ getTypes(Archive *fout, int *numTypes) "typnamespace, typacl, NULL as rtypacl, " "NULL AS inittypacl, NULL AS initrtypacl, " "(%s typowner) AS rolname, " - "typinput::oid AS typinput, " - "typoutput::oid AS typoutput, typelem, typrelid, " + "typelem, typrelid, " "CASE WHEN typrelid = 0 THEN ' '::\"char\" " "ELSE (SELECT relkind FROM pg_class WHERE oid = typrelid) END AS typrelkind, " "typtype, typisdefined, " @@ -3779,8 +3775,7 @@ getTypes(Archive *fout, int *numTypes) "typnamespace, NULL AS typacl, NULL as rtypacl, " "NULL AS inittypacl, NULL AS initrtypacl, " "(%s typowner) AS rolname, " - "typinput::oid AS typinput, " - "typoutput::oid AS typoutput, typelem, typrelid, " + "typelem, typrelid, " "CASE WHEN typrelid = 0 THEN ' '::\"char\" " "ELSE (SELECT relkind FROM pg_class WHERE oid = typrelid) END AS typrelkind, " "typtype, typisdefined, " @@ -3795,8 +3790,7 @@ getTypes(Archive *fout, int *numTypes) "typnamespace, NULL AS typacl, NULL as rtypacl, " "NULL AS inittypacl, NULL AS initrtypacl, " "(%s typowner) AS rolname, " - "typinput::oid AS typinput, " - "typoutput::oid AS typoutput, typelem, typrelid, " + "typelem, typrelid, " "CASE WHEN typrelid = 0 THEN ' '::\"char\" " "ELSE (SELECT relkind FROM pg_class WHERE oid = typrelid) END AS typrelkind, " "typtype, typisdefined, " @@ -3820,8 +3814,6 @@ getTypes(Archive *fout, int *numTypes) i_inittypacl = PQfnumber(res, "inittypacl"); i_initrtypacl = PQfnumber(res, "initrtypacl"); i_rolname = PQfnumber(res, "rolname"); - i_typinput = PQfnumber(res, "typinput"); - i_typoutput = PQfnumber(res, "typoutput"); i_typelem = PQfnumber(res, "typelem"); i_typrelid = PQfnumber(res, "typrelid"); i_typrelkind = PQfnumber(res, "typrelkind"); From 4f52fd3c6d28a4380a5afc51ae6f774c91837a38 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 12 Oct 2016 18:01:43 -0400 Subject: [PATCH 314/871] Revert addition of PGDLLEXPORT in PG_FUNCTION_INFO_V1 macro. This turns out not to be as harmless as I thought: MSVC will complain if it sees an "extern" declaration without PGDLLEXPORT and then one with. (Seems fairly silly, given that this can be changed after the fact by the linker, but there you have it.) Therefore, contrib modules that have extern's for V1 functions in header files are falling over in the buildfarm, since none of those externs are marked PGDLLEXPORT. We might or might not conclude that we're willing to plaster those declarations with PGDLLEXPORT in HEAD, but in any case there's no way we're going to ship this change in the back branches. Third-party authors would not thank us for breaking their code in a minor release. Hence, revert the addition of PGDLLEXPORT (but let's keep the extra info in the comment). If we do the other changes we can revert this commit in HEAD. Per buildfarm. --- src/include/fmgr.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/include/fmgr.h b/src/include/fmgr.h index 18c224ecfa..0878418516 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -350,11 +350,12 @@ typedef const Pg_finfo_record *(*PGFInfoFunction) (void); * * On Windows, the function and info function must be exported. Our normal * build processes take care of that via .DEF files or --export-all-symbols. - * We add PGDLLEXPORT nonetheless so that C functions built with a - * different build process are guaranteed to be exported. + * Module authors using a different build process might need to manually + * declare the function PGDLLEXPORT. We do that automatically here for the + * info function, since authors shouldn't need to be explicitly aware of it. */ #define PG_FUNCTION_INFO_V1(funcname) \ -extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); \ +extern Datum funcname(PG_FUNCTION_ARGS); \ extern PGDLLEXPORT const Pg_finfo_record * CppConcat(pg_finfo_,funcname)(void); \ const Pg_finfo_record * \ CppConcat(pg_finfo_,funcname) (void) \ From 248776ea06c240ae4605e77369d66bcd7ae4f9e3 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 12 Oct 2016 17:01:19 -0700 Subject: [PATCH 315/871] Remove spurious word. Tatsuo Ishii --- doc/src/sgml/parallel.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index c80d42dbef..1e71529eeb 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -241,7 +241,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; than normal but would produce incorrect results. Instead, the parallel portion of the plan must be what is known internally to the query optimizer as a partial plan; that is, it must constructed - so that each process will which executes the plan will generate only a + so that each process which executes the plan will generate only a subset of the output rows in such a way that each required output row is guaranteed to be generated by exactly one of the cooperating processes. From b4fc645787cc7c614c0c97fc9fffacf2bdc6a388 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 12 Oct 2016 18:29:57 -0700 Subject: [PATCH 316/871] Make pg_dumpall's database ACL query independent of hash table order. Previously GRANT order on databases was not well defined, due to the use of EXCEPT without an ORDER BY. Add an ORDER BY, adapt test output. I don't, at the moment, see reason to backpatch this. --- src/bin/pg_dump/pg_dumpall.c | 10 ++++++---- src/bin/pg_dump/t/002_pg_dump.pl | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 82157e5620..45749f3e64 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -1307,11 +1307,13 @@ dumpCreateDB(PGconn *conn) "pg_encoding_to_char(d.encoding), " "datcollate, datctype, datfrozenxid, datminmxid, " "datistemplate, " - "(SELECT pg_catalog.array_agg(acl) FROM (SELECT pg_catalog.unnest(coalesce(datacl,pg_catalog.acldefault('d',datdba))) AS acl " - "EXCEPT SELECT pg_catalog.unnest(pg_catalog.acldefault('d',datdba))) as foo)" + "(SELECT pg_catalog.array_agg(acl ORDER BY acl::text) FROM ( " + " SELECT pg_catalog.unnest(coalesce(datacl,pg_catalog.acldefault('d',datdba))) AS acl " + " EXCEPT SELECT pg_catalog.unnest(pg_catalog.acldefault('d',datdba))) as datacls)" "AS datacl, " - "(SELECT pg_catalog.array_agg(acl) FROM (SELECT pg_catalog.unnest(pg_catalog.acldefault('d',datdba)) AS acl " - "EXCEPT SELECT pg_catalog.unnest(coalesce(datacl,pg_catalog.acldefault('d',datdba)))) as foo)" + "(SELECT pg_catalog.array_agg(acl ORDER BY acl::text) FROM ( " + " SELECT pg_catalog.unnest(pg_catalog.acldefault('d',datdba)) AS acl " + " EXCEPT SELECT pg_catalog.unnest(coalesce(datacl,pg_catalog.acldefault('d',datdba)))) as rdatacls)" "AS rdatacl, " "datconnlimit, " "(SELECT spcname FROM pg_tablespace t WHERE t.oid = d.dattablespace) AS dattablespace " diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 37cbdcdaa5..1d3af87334 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -2484,6 +2484,7 @@ create_sql => 'REVOKE CONNECT ON DATABASE dump_test FROM public;', regexp => qr/^ \QREVOKE CONNECT,TEMPORARY ON DATABASE dump_test FROM PUBLIC;\E\n + \QGRANT CREATE ON DATABASE dump_test TO regress_dump_test_role;\E\n \QGRANT TEMPORARY ON DATABASE dump_test TO PUBLIC;\E /xm, like => { pg_dumpall_dbprivs => 1, }, From ccbb852cd6101e93976111676a90f1e5d9268951 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 12 Oct 2016 18:31:45 -0700 Subject: [PATCH 317/871] Fix further hash table order dependent tests. Similar to 0137caf273, this makes contrib and pl tests less dependant on hash-table order. After this commit, at least some order affecting changes to execGrouping.c don't result in regression test changes anymore. --- contrib/test_decoding/expected/ddl.out | 4 ++-- contrib/test_decoding/sql/ddl.sql | 4 ++-- src/pl/plpython/expected/plpython_spi.out | 4 ++-- src/pl/plpython/sql/plpython_spi.sql | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/contrib/test_decoding/expected/ddl.out b/contrib/test_decoding/expected/ddl.out index bd9b42f401..7fbeafdb39 100644 --- a/contrib/test_decoding/expected/ddl.out +++ b/contrib/test_decoding/expected/ddl.out @@ -274,9 +274,9 @@ INSERT INTO tr_etoomuch (id, data) SELECT g.i, -g.i FROM generate_series(8000, 12000) g(i) ON CONFLICT(id) DO UPDATE SET data = EXCLUDED.data; SELECT substring(data, 1, 29), count(*) -FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1') +FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1') WITH ORDINALITY GROUP BY 1 -ORDER BY min(location - '0/0'); +ORDER BY min(ordinality); substring | count -------------------------------+------- BEGIN | 1 diff --git a/contrib/test_decoding/sql/ddl.sql b/contrib/test_decoding/sql/ddl.sql index e99b2d37d9..89b8b8f780 100644 --- a/contrib/test_decoding/sql/ddl.sql +++ b/contrib/test_decoding/sql/ddl.sql @@ -146,9 +146,9 @@ SELECT g.i, -g.i FROM generate_series(8000, 12000) g(i) ON CONFLICT(id) DO UPDATE SET data = EXCLUDED.data; SELECT substring(data, 1, 29), count(*) -FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1') +FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1') WITH ORDINALITY GROUP BY 1 -ORDER BY min(location - '0/0'); +ORDER BY min(ordinality); /* * check whether we decode subtransactions correctly in relation with each diff --git a/src/pl/plpython/expected/plpython_spi.out b/src/pl/plpython/expected/plpython_spi.out index dbde36f841..0d78ca1de4 100644 --- a/src/pl/plpython/expected/plpython_spi.out +++ b/src/pl/plpython/expected/plpython_spi.out @@ -220,8 +220,8 @@ SELECT result_len_test($$UPDATE foo3 SET b= '' WHERE a = 2$$); CREATE FUNCTION result_subscript_test() RETURNS void AS $$ -result = plpy.execute("SELECT 1 AS c UNION SELECT 2 " - "UNION SELECT 3 UNION SELECT 4") +result = plpy.execute("SELECT 1 AS c UNION ALL SELECT 2 " + "UNION ALL SELECT 3 UNION ALL SELECT 4") plpy.info(result[1]['c']) plpy.info(result[-1]['c']) diff --git a/src/pl/plpython/sql/plpython_spi.sql b/src/pl/plpython/sql/plpython_spi.sql index 87170609da..7427de824b 100644 --- a/src/pl/plpython/sql/plpython_spi.sql +++ b/src/pl/plpython/sql/plpython_spi.sql @@ -135,8 +135,8 @@ SELECT result_len_test($$UPDATE foo3 SET b= '' WHERE a = 2$$); CREATE FUNCTION result_subscript_test() RETURNS void AS $$ -result = plpy.execute("SELECT 1 AS c UNION SELECT 2 " - "UNION SELECT 3 UNION SELECT 4") +result = plpy.execute("SELECT 1 AS c UNION ALL SELECT 2 " + "UNION ALL SELECT 3 UNION ALL SELECT 4") plpy.info(result[1]['c']) plpy.info(result[-1]['c']) From 9c4cc9e2c75e84a8267396be5cccbfe25b8f63f6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 13 Oct 2016 00:25:28 -0400 Subject: [PATCH 318/871] Fix broken jsonb_set() logic for replacing array elements. Commit 0b62fd036 did a fairly sloppy job of refactoring setPath() to support jsonb_insert() along with jsonb_set(). In its defense, though, there was no regression test case exercising the case of replacing an existing element in a jsonb array. Per bug #14366 from Peng Sun. Back-patch to 9.6 where bug was introduced. Report: <20161012065349.1412.47858@wrigleys.postgresql.org> --- src/backend/utils/adt/jsonfuncs.c | 12 ++++++------ src/test/regress/expected/jsonb.out | 6 ++++++ src/test/regress/sql/jsonb.sql | 1 + 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c index 996007d483..059d570de7 100644 --- a/src/backend/utils/adt/jsonfuncs.c +++ b/src/backend/utils/adt/jsonfuncs.c @@ -34,11 +34,11 @@ #include "utils/typcache.h" /* Operations available for setPath */ -#define JB_PATH_NOOP 0x0000 #define JB_PATH_CREATE 0x0001 #define JB_PATH_DELETE 0x0002 -#define JB_PATH_INSERT_BEFORE 0x0004 -#define JB_PATH_INSERT_AFTER 0x0008 +#define JB_PATH_REPLACE 0x0004 +#define JB_PATH_INSERT_BEFORE 0x0008 +#define JB_PATH_INSERT_AFTER 0x0010 #define JB_PATH_CREATE_OR_INSERT \ (JB_PATH_INSERT_BEFORE | JB_PATH_INSERT_AFTER | JB_PATH_CREATE) @@ -3545,7 +3545,7 @@ jsonb_set(PG_FUNCTION_ARGS) it = JsonbIteratorInit(&in->root); res = setPath(&it, path_elems, path_nulls, path_len, &st, - 0, newval, create ? JB_PATH_CREATE : JB_PATH_NOOP); + 0, newval, create ? JB_PATH_CREATE : JB_PATH_REPLACE); Assert(res != NULL); @@ -4003,7 +4003,7 @@ setPathArray(JsonbIterator **it, Datum *path_elems, bool *path_nulls, if (op_type & (JB_PATH_INSERT_AFTER | JB_PATH_INSERT_BEFORE)) (void) pushJsonbValue(st, r, &v); - if (op_type & JB_PATH_INSERT_AFTER) + if (op_type & (JB_PATH_INSERT_AFTER | JB_PATH_REPLACE)) addJsonbToParseState(st, newval); done = true; @@ -4035,7 +4035,7 @@ setPathArray(JsonbIterator **it, Datum *path_elems, bool *path_nulls, } } - if (op_type & JB_PATH_CREATE_OR_INSERT && !done && + if ((op_type & JB_PATH_CREATE_OR_INSERT) && !done && level == path_len - 1 && i == nelems - 1) { addJsonbToParseState(st, newval); diff --git a/src/test/regress/expected/jsonb.out b/src/test/regress/expected/jsonb.out index a6d25defb0..e2cb08a6fb 100644 --- a/src/test/regress/expected/jsonb.out +++ b/src/test/regress/expected/jsonb.out @@ -3238,6 +3238,12 @@ select jsonb_set('[]','{1}','"b"', false); [] (1 row) +select jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0}','[2,3,4]', false); + jsonb_set +------------------------- + [[2, 3, 4], 2, null, 3] +(1 row) + -- jsonb_set adding instead of replacing -- prepend to array select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{b,-33}','{"foo":123}'); diff --git a/src/test/regress/sql/jsonb.sql b/src/test/regress/sql/jsonb.sql index b84bd70a29..6b4c796992 100644 --- a/src/test/regress/sql/jsonb.sql +++ b/src/test/regress/sql/jsonb.sql @@ -814,6 +814,7 @@ select '[]'::jsonb #- '{a}'; select jsonb_set('"a"','{a}','"b"'); --error select jsonb_set('{}','{a}','"b"', false); select jsonb_set('[]','{1}','"b"', false); +select jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0}','[2,3,4]', false); -- jsonb_set adding instead of replacing From 0a4bf6b192377bef8e69c92d0c95434a91509f12 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 13 Oct 2016 10:46:22 -0400 Subject: [PATCH 319/871] Fix pg_dumpall regression test to be locale-independent. The expected results in commit b4fc64578 seem to have been generated in a non-C locale, which just points up the fact that the ORDER BY clause was locale-sensitive. Per buildfarm. --- src/bin/pg_dump/pg_dumpall.c | 4 ++-- src/bin/pg_dump/t/002_pg_dump.pl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 45749f3e64..48bfca70ec 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -1307,11 +1307,11 @@ dumpCreateDB(PGconn *conn) "pg_encoding_to_char(d.encoding), " "datcollate, datctype, datfrozenxid, datminmxid, " "datistemplate, " - "(SELECT pg_catalog.array_agg(acl ORDER BY acl::text) FROM ( " + "(SELECT pg_catalog.array_agg(acl ORDER BY acl::text COLLATE \"C\") FROM ( " " SELECT pg_catalog.unnest(coalesce(datacl,pg_catalog.acldefault('d',datdba))) AS acl " " EXCEPT SELECT pg_catalog.unnest(pg_catalog.acldefault('d',datdba))) as datacls)" "AS datacl, " - "(SELECT pg_catalog.array_agg(acl ORDER BY acl::text) FROM ( " + "(SELECT pg_catalog.array_agg(acl ORDER BY acl::text COLLATE \"C\") FROM ( " " SELECT pg_catalog.unnest(pg_catalog.acldefault('d',datdba)) AS acl " " EXCEPT SELECT pg_catalog.unnest(coalesce(datacl,pg_catalog.acldefault('d',datdba)))) as rdatacls)" "AS rdatacl, " diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 1d3af87334..90f075bf4f 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -2484,8 +2484,8 @@ create_sql => 'REVOKE CONNECT ON DATABASE dump_test FROM public;', regexp => qr/^ \QREVOKE CONNECT,TEMPORARY ON DATABASE dump_test FROM PUBLIC;\E\n - \QGRANT CREATE ON DATABASE dump_test TO regress_dump_test_role;\E\n - \QGRANT TEMPORARY ON DATABASE dump_test TO PUBLIC;\E + \QGRANT TEMPORARY ON DATABASE dump_test TO PUBLIC;\E\n + \QGRANT CREATE ON DATABASE dump_test TO regress_dump_test_role;\E /xm, like => { pg_dumpall_dbprivs => 1, }, unlike => { From 15fc5e15811337f5a81d4ae44c6149256f6dd15f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 13 Oct 2016 13:59:56 -0400 Subject: [PATCH 320/871] Clean up handling of anonymous mmap'd shared-memory segment. Fix detaching of the mmap'd segment to have its own on_shmem_exit callback, rather than piggybacking on the one for detaching from the SysV segment. That was confusing, and given the distance between the two attach calls, it was trouble waiting to happen. Make the detaching calls idempotent by clearing AnonymousShmem to show we've already unmapped. I spent quite a bit of time yesterday trying to find a path that would allow the munmap()'s to be done twice, and while I did not succeed, it seems silly that there's even a question. Make the #ifdef logic less confusing by separating "do we want to use anonymous shmem" from EXEC_BACKEND. Even though there's no current scenario where those conditions are different, it is not helpful for different places in the same file to be testing EXEC_BACKEND for what are fundamentally different reasons. Don't do on_exit_reset() in StartBackgroundWorker(). At best that's useless (InitPostmasterChild would have done it already) and at worst it could zap some callback that's unrelated to shared memory. Improve comments, and simplify the huge_pages enablement logic slightly. Back-patch to 9.4 where hugepage support was introduced. Arguably this should go into 9.3 as well, but the code looks significantly different there, and I doubt it's worth the trouble of adapting the patch given I can't show a live bug. --- src/backend/port/sysv_shmem.c | 127 +++++++++++++++++++----------- src/backend/postmaster/bgworker.c | 1 - 2 files changed, 82 insertions(+), 46 deletions(-) diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c index 6c442b927a..31972d4f6c 100644 --- a/src/backend/port/sysv_shmem.c +++ b/src/backend/port/sysv_shmem.c @@ -3,8 +3,11 @@ * sysv_shmem.c * Implement shared memory using SysV facilities * - * These routines represent a fairly thin layer on top of SysV shared - * memory functionality. + * These routines used to be a fairly thin layer on top of SysV shared + * memory functionality. With the addition of anonymous-shmem logic, + * they're a bit fatter now. We still require a SysV shmem block to + * exist, though, because mmap'd shmem provides no way to find out how + * many processes are attached, which we need for interlocking purposes. * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -36,14 +39,44 @@ #include "utils/guc.h" +/* + * As of PostgreSQL 9.3, we normally allocate only a very small amount of + * System V shared memory, and only for the purposes of providing an + * interlock to protect the data directory. The real shared memory block + * is allocated using mmap(). This works around the problem that many + * systems have very low limits on the amount of System V shared memory + * that can be allocated. Even a limit of a few megabytes will be enough + * to run many copies of PostgreSQL without needing to adjust system settings. + * + * We assume that no one will attempt to run PostgreSQL 9.3 or later on + * systems that are ancient enough that anonymous shared memory is not + * supported, such as pre-2.4 versions of Linux. If that turns out to be + * false, we might need to add compile and/or run-time tests here and do this + * only if the running kernel supports it. + * + * However, we must always disable this logic in the EXEC_BACKEND case, and + * fall back to the old method of allocating the entire segment using System V + * shared memory, because there's no way to attach an anonymous mmap'd segment + * to a process after exec(). Since EXEC_BACKEND is intended only for + * developer use, this shouldn't be a big problem. Because of this, we do + * not worry about supporting anonymous shmem in the EXEC_BACKEND cases below. + */ +#ifndef EXEC_BACKEND +#define USE_ANONYMOUS_SHMEM +#endif + + typedef key_t IpcMemoryKey; /* shared memory key passed to shmget(2) */ typedef int IpcMemoryId; /* shared memory ID returned by shmget(2) */ unsigned long UsedShmemSegID = 0; void *UsedShmemSegAddr = NULL; + +#ifdef USE_ANONYMOUS_SHMEM static Size AnonymousShmemSize; static void *AnonymousShmem = NULL; +#endif static void *InternalIpcMemoryCreate(IpcMemoryKey memKey, Size size); static void IpcMemoryDetach(int status, Datum shmaddr); @@ -204,10 +237,6 @@ IpcMemoryDetach(int status, Datum shmaddr) /* Detach System V shared memory block. */ if (shmdt(DatumGetPointer(shmaddr)) < 0) elog(LOG, "shmdt(%p) failed: %m", DatumGetPointer(shmaddr)); - /* Release anonymous shared memory block, if any. */ - if (AnonymousShmem != NULL - && munmap(AnonymousShmem, AnonymousShmemSize) < 0) - elog(LOG, "munmap(%p) failed: %m", AnonymousShmem); } /****************************************************************************/ @@ -318,6 +347,8 @@ PGSharedMemoryIsInUse(unsigned long id1, unsigned long id2) return true; } +#ifdef USE_ANONYMOUS_SHMEM + /* * Creates an anonymous mmap()ed shared memory segment. * @@ -325,7 +356,6 @@ PGSharedMemoryIsInUse(unsigned long id1, unsigned long id2) * actual size of the allocation, if it ends up allocating a segment that is * larger than requested. */ -#ifndef EXEC_BACKEND static void * CreateAnonymousSegment(Size *size) { @@ -334,10 +364,8 @@ CreateAnonymousSegment(Size *size) int mmap_errno = 0; #ifndef MAP_HUGETLB - if (huge_pages == HUGE_PAGES_ON) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("huge TLB pages not supported on this platform"))); + /* PGSharedMemoryCreate should have dealt with this case */ + Assert(huge_pages != HUGE_PAGES_ON); #else if (huge_pages == HUGE_PAGES_ON || huge_pages == HUGE_PAGES_TRY) { @@ -366,12 +394,12 @@ CreateAnonymousSegment(Size *size) PG_MMAP_FLAGS | MAP_HUGETLB, -1, 0); mmap_errno = errno; if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED) - elog(DEBUG1, "mmap with MAP_HUGETLB failed, huge pages disabled: %m"); + elog(DEBUG1, "mmap(%zu) with MAP_HUGETLB failed, huge pages disabled: %m", + allocsize); } #endif - if (huge_pages == HUGE_PAGES_OFF || - (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)) + if (ptr == MAP_FAILED && huge_pages != HUGE_PAGES_ON) { /* * use the original size, not the rounded up value, when falling back @@ -401,7 +429,25 @@ CreateAnonymousSegment(Size *size) *size = allocsize; return ptr; } -#endif + +/* + * AnonymousShmemDetach --- detach from an anonymous mmap'd block + * (called as an on_shmem_exit callback, hence funny argument list) + */ +static void +AnonymousShmemDetach(int status, Datum arg) +{ + /* Release anonymous shared memory block, if any. */ + if (AnonymousShmem != NULL) + { + if (munmap(AnonymousShmem, AnonymousShmemSize) < 0) + elog(LOG, "munmap(%p, %zu) failed: %m", + AnonymousShmem, AnonymousShmemSize); + AnonymousShmem = NULL; + } +} + +#endif /* USE_ANONYMOUS_SHMEM */ /* * PGSharedMemoryCreate @@ -432,7 +478,8 @@ PGSharedMemoryCreate(Size size, bool makePrivate, int port, struct stat statbuf; Size sysvsize; -#if defined(EXEC_BACKEND) || !defined(MAP_HUGETLB) + /* Complain if hugepages demanded but we can't possibly support them */ +#if !defined(USE_ANONYMOUS_SHMEM) || !defined(MAP_HUGETLB) if (huge_pages == HUGE_PAGES_ON) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -442,32 +489,13 @@ PGSharedMemoryCreate(Size size, bool makePrivate, int port, /* Room for a header? */ Assert(size > MAXALIGN(sizeof(PGShmemHeader))); - /* - * As of PostgreSQL 9.3, we normally allocate only a very small amount of - * System V shared memory, and only for the purposes of providing an - * interlock to protect the data directory. The real shared memory block - * is allocated using mmap(). This works around the problem that many - * systems have very low limits on the amount of System V shared memory - * that can be allocated. Even a limit of a few megabytes will be enough - * to run many copies of PostgreSQL without needing to adjust system - * settings. - * - * We assume that no one will attempt to run PostgreSQL 9.3 or later on - * systems that are ancient enough that anonymous shared memory is not - * supported, such as pre-2.4 versions of Linux. If that turns out to be - * false, we might need to add a run-time test here and do this only if - * the running kernel supports it. - * - * However, we disable this logic in the EXEC_BACKEND case, and fall back - * to the old method of allocating the entire segment using System V - * shared memory, because there's no way to attach an mmap'd segment to a - * process after exec(). Since EXEC_BACKEND is intended only for - * developer use, this shouldn't be a big problem. - */ -#ifndef EXEC_BACKEND +#ifdef USE_ANONYMOUS_SHMEM AnonymousShmem = CreateAnonymousSegment(&size); AnonymousShmemSize = size; + /* Register on-exit routine to unmap the anonymous segment */ + on_shmem_exit(AnonymousShmemDetach, (Datum) 0); + /* Now we need only allocate a minimal-sized SysV shmem block. */ sysvsize = sizeof(PGShmemHeader); #else @@ -572,10 +600,14 @@ PGSharedMemoryCreate(Size size, bool makePrivate, int port, * block. Otherwise, the System V shared memory block is only a shim, and * we must return a pointer to the real block. */ +#ifdef USE_ANONYMOUS_SHMEM if (AnonymousShmem == NULL) return hdr; memcpy(AnonymousShmem, hdr, sizeof(PGShmemHeader)); return (PGShmemHeader *) AnonymousShmem; +#else + return hdr; +#endif } #ifdef EXEC_BACKEND @@ -660,12 +692,12 @@ PGSharedMemoryNoReAttach(void) * * Detach from the shared memory segment, if still attached. This is not * intended to be called explicitly by the process that originally created the - * segment (it will have an on_shmem_exit callback registered to do that). + * segment (it will have on_shmem_exit callback(s) registered to do that). * Rather, this is for subprocesses that have inherited an attachment and want * to get rid of it. * * UsedShmemSegID and UsedShmemSegAddr are implicit parameters to this - * routine. + * routine, also AnonymousShmem and AnonymousShmemSize. */ void PGSharedMemoryDetach(void) @@ -682,10 +714,15 @@ PGSharedMemoryDetach(void) UsedShmemSegAddr = NULL; } - /* Release anonymous shared memory block, if any. */ - if (AnonymousShmem != NULL - && munmap(AnonymousShmem, AnonymousShmemSize) < 0) - elog(LOG, "munmap(%p) failed: %m", AnonymousShmem); +#ifdef USE_ANONYMOUS_SHMEM + if (AnonymousShmem != NULL) + { + if (munmap(AnonymousShmem, AnonymousShmemSize) < 0) + elog(LOG, "munmap(%p, %zu) failed: %m", + AnonymousShmem, AnonymousShmemSize); + AnonymousShmem = NULL; + } +#endif } diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 3a2929a00b..8f6255432e 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -602,7 +602,6 @@ StartBackgroundWorker(void) */ if ((worker->bgw_flags & BGWORKER_SHMEM_ACCESS) == 0) { - on_exit_reset(); dsm_detach_all(); PGSharedMemoryDetach(); } From cb775768e3e37d466d69b7177a92508b81c1c204 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 13 Oct 2016 15:06:46 -0400 Subject: [PATCH 321/871] Try to find out the actual hugepage size when making a MAP_HUGETLB request. Even if Linux's mmap() is okay with a partial-hugepage request, munmap() is not, as reported by Chris Richards. Therefore it behooves us to try a bit harder to find out the actual hugepage size, instead of assuming that we can skate by with a guess. For the moment, just look into /proc/meminfo to find out the default hugepage size, and use that. Later, on kernels that support requests for nondefault sizes, we might try to consider other alternatives. But that smells more like a new feature than a bug fix, especially if we want to provide any way for the DBA to control it, so leave it for another day. I set this up to allow easy addition of platform-specific code for non-Linux platforms, if needed; but right now there are no reports suggesting that we need to work harder on other platforms. Back-patch to 9.4 where hugepage support was introduced. Discussion: <31056.1476303954@sss.pgh.pa.us> --- src/backend/port/sysv_shmem.c | 97 +++++++++++++++++++++++++++++------ 1 file changed, 81 insertions(+), 16 deletions(-) diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c index 31972d4f6c..1ad898d350 100644 --- a/src/backend/port/sysv_shmem.c +++ b/src/backend/port/sysv_shmem.c @@ -34,6 +34,7 @@ #include "miscadmin.h" #include "portability/mem.h" #include "storage/dsm.h" +#include "storage/fd.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" #include "utils/guc.h" @@ -349,6 +350,80 @@ PGSharedMemoryIsInUse(unsigned long id1, unsigned long id2) #ifdef USE_ANONYMOUS_SHMEM +#ifdef MAP_HUGETLB + +/* + * Identify the huge page size to use. + * + * Some Linux kernel versions have a bug causing mmap() to fail on requests + * that are not a multiple of the hugepage size. Versions without that bug + * instead silently round the request up to the next hugepage multiple --- + * and then munmap() fails when we give it a size different from that. + * So we have to round our request up to a multiple of the actual hugepage + * size to avoid trouble. + * + * Doing the round-up ourselves also lets us make use of the extra memory, + * rather than just wasting it. Currently, we just increase the available + * space recorded in the shmem header, which will make the extra usable for + * purposes such as additional locktable entries. Someday, for very large + * hugepage sizes, we might want to think about more invasive strategies, + * such as increasing shared_buffers to absorb the extra space. + * + * Returns the (real or assumed) page size into *hugepagesize, + * and the hugepage-related mmap flags to use into *mmap_flags. + * + * Currently *mmap_flags is always just MAP_HUGETLB. Someday, on systems + * that support it, we might OR in additional bits to specify a particular + * non-default huge page size. + */ +static void +GetHugePageSize(Size *hugepagesize, int *mmap_flags) +{ + /* + * If we fail to find out the system's default huge page size, assume it + * is 2MB. This will work fine when the actual size is less. If it's + * more, we might get mmap() or munmap() failures due to unaligned + * requests; but at this writing, there are no reports of any non-Linux + * systems being picky about that. + */ + *hugepagesize = 2 * 1024 * 1024; + *mmap_flags = MAP_HUGETLB; + + /* + * System-dependent code to find out the default huge page size. + * + * On Linux, read /proc/meminfo looking for a line like "Hugepagesize: + * nnnn kB". Ignore any failures, falling back to the preset default. + */ +#ifdef __linux__ + { + FILE *fp = AllocateFile("/proc/meminfo", "r"); + char buf[128]; + unsigned int sz; + char ch; + + if (fp) + { + while (fgets(buf, sizeof(buf), fp)) + { + if (sscanf(buf, "Hugepagesize: %u %c", &sz, &ch) == 2) + { + if (ch == 'k') + { + *hugepagesize = sz * (Size) 1024; + break; + } + /* We could accept other units besides kB, if needed */ + } + } + FreeFile(fp); + } + } +#endif /* __linux__ */ +} + +#endif /* MAP_HUGETLB */ + /* * Creates an anonymous mmap()ed shared memory segment. * @@ -371,27 +446,17 @@ CreateAnonymousSegment(Size *size) { /* * Round up the request size to a suitable large value. - * - * Some Linux kernel versions are known to have a bug, which causes - * mmap() with MAP_HUGETLB to fail if the request size is not a - * multiple of any supported huge page size. To work around that, we - * round up the request size to nearest 2MB. 2MB is the most common - * huge page page size on affected systems. - * - * Aside from that bug, even with a kernel that does the allocation - * correctly, rounding it up ourselves avoids wasting memory. Without - * it, if we for example make an allocation of 2MB + 1 bytes, the - * kernel might decide to use two 2MB huge pages for that, and waste 2 - * MB - 1 of memory. When we do the rounding ourselves, we can use - * that space for allocations. */ - int hugepagesize = 2 * 1024 * 1024; + Size hugepagesize; + int mmap_flags; + + GetHugePageSize(&hugepagesize, &mmap_flags); if (allocsize % hugepagesize != 0) allocsize += hugepagesize - (allocsize % hugepagesize); ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE, - PG_MMAP_FLAGS | MAP_HUGETLB, -1, 0); + PG_MMAP_FLAGS | mmap_flags, -1, 0); mmap_errno = errno; if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED) elog(DEBUG1, "mmap(%zu) with MAP_HUGETLB failed, huge pages disabled: %m", @@ -402,7 +467,7 @@ CreateAnonymousSegment(Size *size) if (ptr == MAP_FAILED && huge_pages != HUGE_PAGES_ON) { /* - * use the original size, not the rounded up value, when falling back + * Use the original size, not the rounded-up value, when falling back * to non-huge pages. */ allocsize = *size; From c08521eb55135c493cee23541c233870cdff14b7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 13 Oct 2016 16:08:16 -0400 Subject: [PATCH 322/871] Remove dead code in pg_dump. I'm not sure if this provision for "pg_backup" behaving a bit differently from "pg_dump" ever did anything useful in a released version. But it's definitely dead code now. Michael Paquier --- src/bin/pg_dump/pg_dump.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 28ec5bf383..4da297f1d0 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -363,10 +363,6 @@ main(int argc, char **argv) progname = get_progname(argv[0]); - /* Set default options based on progname */ - if (strcmp(progname, "pg_backup") == 0) - format = "c"; - if (argc > 1) { if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) From 3cca13cbfcc5312f7ae1728213e197c6f37ca62a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 13 Oct 2016 17:05:14 -0400 Subject: [PATCH 323/871] Fix another bug in merging of inherited CHECK constraints. It's not good for an inherited child constraint to be marked connoinherit; that would result in the constraint not propagating to grandchild tables, if any are created later. The code mostly prevented this from happening but there was one case that was missed. This is somewhat related to commit e55a946a8, which also tightened checks on constraint merging. Hence, back-patch to 9.2 like that one. This isn't so much because there's a concrete feature-related reason to stop there, as to avoid having more distinct behaviors than we have to in this area. Amit Langote Discussion: --- src/backend/catalog/heap.c | 11 +++++++++++ src/test/regress/expected/inherit.out | 3 +++ src/test/regress/sql/inherit.sql | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index ea06a5739a..0cf7b9eb62 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -2462,6 +2462,17 @@ MergeWithExistingConstraint(Relation rel, char *ccname, Node *expr, errmsg("constraint \"%s\" conflicts with non-inherited constraint on relation \"%s\"", ccname, RelationGetRelationName(rel)))); + /* + * Must not change an existing inherited constraint to "no + * inherit" status. That's because inherited constraints should + * be able to propagate to lower-level children. + */ + if (con->coninhcount > 0 && is_no_inherit) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("constraint \"%s\" conflicts with inherited constraint on relation \"%s\"", + ccname, RelationGetRelationName(rel)))); + /* * If the child constraint is "not valid" then cannot merge with a * valid parent constraint diff --git a/src/test/regress/expected/inherit.out b/src/test/regress/expected/inherit.out index 9d374fe6c4..df7cba6614 100644 --- a/src/test/regress/expected/inherit.out +++ b/src/test/regress/expected/inherit.out @@ -645,6 +645,9 @@ Check constraints: "p2chk" CHECK (ff1 > 10) Inherits: p1 +-- Test that child does not override inheritable constraints of the parent +create table c2 (constraint p2chk check (ff1 > 10) no inherit) inherits (p1); --fails +ERROR: constraint "p2chk" conflicts with inherited constraint on relation "c2" drop table p1 cascade; NOTICE: drop cascades to table c1 -- Tests for casting between the rowtypes of parent and child diff --git a/src/test/regress/sql/inherit.sql b/src/test/regress/sql/inherit.sql index 6b1df754a6..f45aab1ac6 100644 --- a/src/test/regress/sql/inherit.sql +++ b/src/test/regress/sql/inherit.sql @@ -157,6 +157,9 @@ create table c1 () inherits (p1); \d p1 \d c1 +-- Test that child does not override inheritable constraints of the parent +create table c2 (constraint p2chk check (ff1 > 10) no inherit) inherits (p1); --fails + drop table p1 cascade; -- Tests for casting between the rowtypes of parent and child From b1ee762a6138df073d4b2b80c235dd9025a8532c Mon Sep 17 00:00:00 2001 From: Tatsuo Ishii Date: Fri, 14 Oct 2016 07:45:25 +0900 Subject: [PATCH 324/871] Fix typo. Confirmed by Tom Lane. --- doc/src/sgml/parallel.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index 1e71529eeb..94170b770b 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -189,7 +189,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; Even when parallel query plan is generated for a particular query, there are several circumstances under which it will be impossible to execute that plan in parallel at execution time. If this occurs, the leader - will execute the portion of the plan between below the Gather + will execute the portion of the plan below the Gather node entirely by itself, almost as if the Gather node were not present. This will happen if any of the following conditions are met: From 81e82a2bd48865f9a294b63d9492f9fde6a32787 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 13 Oct 2016 19:45:58 -0400 Subject: [PATCH 325/871] Fix handling of pgstat counters for TRUNCATE in a prepared transaction. pgstat_twophase_postcommit is supposed to duplicate the math in AtEOXact_PgStat, but it had missed out the bit about clearing t_delta_live_tuples/t_delta_dead_tuples for a TRUNCATE. It's harder than you might think to replicate the issue here, because those counters would only be nonzero when a previous transaction in the same backend had added/deleted tuples in the truncated table, and those counts hadn't been sent to the stats collector yet. Evident oversight in commit d42358efb. I've not added a regression test for this; we tried to add one in d42358efb, and had to revert it because it was too timing-sensitive for the buildfarm. Back-patch to 9.5 where d42358efb came in. Stas Kelvich Discussion: --- src/backend/postmaster/pgstat.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index a9efee8304..5112d6d96d 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -2226,7 +2226,12 @@ pgstat_twophase_postcommit(TransactionId xid, uint16 info, pgstat_info->t_counts.t_tuples_updated += rec->tuples_updated; pgstat_info->t_counts.t_tuples_deleted += rec->tuples_deleted; pgstat_info->t_counts.t_truncated = rec->t_truncated; - + if (rec->t_truncated) + { + /* forget live/dead stats seen by backend thus far */ + pgstat_info->t_counts.t_delta_live_tuples = 0; + pgstat_info->t_counts.t_delta_dead_tuples = 0; + } pgstat_info->t_counts.t_delta_live_tuples += rec->tuples_inserted - rec->tuples_deleted; pgstat_info->t_counts.t_delta_dead_tuples += From 13d3180fd14c624bbb274e200e98ddb50e260216 Mon Sep 17 00:00:00 2001 From: Tatsuo Ishii Date: Fri, 14 Oct 2016 09:03:25 +0900 Subject: [PATCH 326/871] Fix typo. Confirmed by Michael Paquier. --- doc/src/sgml/parallel.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index 94170b770b..d0b438e889 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -339,7 +339,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; When executing a parallel plan, you can use EXPLAIN (ANALYZE, - VERBOSE) will display per-worker statistics for each plan node. + VERBOSE) to display per-worker statistics for each plan node. This may be useful in determining whether the work is being evenly distributed between all plan nodes and more generally in understanding the performance characteristics of the plan. From 32fdf42cf546f613aab9ca98935c40a046187fa9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 14 Oct 2016 16:28:34 -0400 Subject: [PATCH 327/871] Fix assorted integer-overflow hazards in varbit.c. bitshiftright() and bitshiftleft() would recursively call each other infinitely if the user passed INT_MIN for the shift amount, due to integer overflow in negating the shift amount. To fix, clamp to -VARBITMAXLEN. That doesn't change the results since any shift distance larger than the input bit string's length produces an all-zeroes result. Also fix some places that seemed inadequately paranoid about input typmods exceeding VARBITMAXLEN. While a typmod accepted by anybit_typmodin() will certainly be much less than that, at least some of these spots are reachable with user-chosen integer values. Andreas Seltenreich and Tom Lane Discussion: <87d1j2zqtz.fsf@credativ.de> --- src/backend/utils/adt/varbit.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/backend/utils/adt/varbit.c b/src/backend/utils/adt/varbit.c index 75e6a46476..397eecc564 100644 --- a/src/backend/utils/adt/varbit.c +++ b/src/backend/utils/adt/varbit.c @@ -305,7 +305,7 @@ bit_recv(PG_FUNCTION_ARGS) bits8 mask; bitlen = pq_getmsgint(buf, sizeof(int32)); - if (bitlen < 0) + if (bitlen < 0 || bitlen > VARBITMAXLEN) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("invalid length in external bit string"))); @@ -368,7 +368,7 @@ bit(PG_FUNCTION_ARGS) bits8 mask; /* No work if typmod is invalid or supplied data matches it already */ - if (len <= 0 || len == VARBITLEN(arg)) + if (len <= 0 || len > VARBITMAXLEN || len == VARBITLEN(arg)) PG_RETURN_VARBIT_P(arg); if (!isExplicit) @@ -621,7 +621,7 @@ varbit_recv(PG_FUNCTION_ARGS) bits8 mask; bitlen = pq_getmsgint(buf, sizeof(int32)); - if (bitlen < 0) + if (bitlen < 0 || bitlen > VARBITMAXLEN) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("invalid length in external bit string"))); @@ -1387,9 +1387,14 @@ bitshiftleft(PG_FUNCTION_ARGS) /* Negative shift is a shift to the right */ if (shft < 0) + { + /* Prevent integer overflow in negation */ + if (shft < -VARBITMAXLEN) + shft = -VARBITMAXLEN; PG_RETURN_DATUM(DirectFunctionCall2(bitshiftright, VarBitPGetDatum(arg), Int32GetDatum(-shft))); + } result = (VarBit *) palloc(VARSIZE(arg)); SET_VARSIZE(result, VARSIZE(arg)); @@ -1447,9 +1452,14 @@ bitshiftright(PG_FUNCTION_ARGS) /* Negative shift is a shift to the left */ if (shft < 0) + { + /* Prevent integer overflow in negation */ + if (shft < -VARBITMAXLEN) + shft = -VARBITMAXLEN; PG_RETURN_DATUM(DirectFunctionCall2(bitshiftleft, VarBitPGetDatum(arg), Int32GetDatum(-shft))); + } result = (VarBit *) palloc(VARSIZE(arg)); SET_VARSIZE(result, VARSIZE(arg)); @@ -1507,7 +1517,7 @@ bitfromint4(PG_FUNCTION_ARGS) int destbitsleft, srcbitsleft; - if (typmod <= 0) + if (typmod <= 0 || typmod > VARBITMAXLEN) typmod = 1; /* default bit length */ rlen = VARBITTOTALLEN(typmod); @@ -1587,7 +1597,7 @@ bitfromint8(PG_FUNCTION_ARGS) int destbitsleft, srcbitsleft; - if (typmod <= 0) + if (typmod <= 0 || typmod > VARBITMAXLEN) typmod = 1; /* default bit length */ rlen = VARBITTOTALLEN(typmod); From aa3ca5e3dd60bf0b992b74f955378f28e601292a Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Fri, 14 Oct 2016 16:05:30 -0700 Subject: [PATCH 328/871] Add likely/unlikely() branch hint macros. These are useful for very hot code paths. Because it's easy to guess wrongly about likelihood, and because such likelihoods change over time, they should be used sparingly. Past tests have shown it'd be a good idea to use them in some places, e.g. in error checks around ereports that ERROR out, but that's work for later. Discussion: <20160727004333.r3e2k2y6fvk2ntup@alap3.anarazel.de> --- src/include/c.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/include/c.h b/src/include/c.h index 4ab3f8027a..672e21e03e 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -939,6 +939,22 @@ typedef NameData *Name; #endif +/* + * Hints to the compiler about the likelihood of a branch. Both likely() and + * unlikely() return the boolean value of the contained expression. + * + * These should only be used sparingly, in very hot code paths. It's very easy + * to mis-estimate likelihoods. + */ +#if __GNUC__ >= 3 +#define likely(x) __builtin_expect((x) != 0, 1) +#define unlikely(x) __builtin_expect((x) != 0, 0) +#else +#define likely(x) ((x) != 0) +#define unlikely(x) ((x) != 0) +#endif + + /* ---------------------------------------------------------------- * Section 8: random stuff * ---------------------------------------------------------------- From b30d3ea824c5ccba43d3e942704f20686e7dbab8 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Fri, 14 Oct 2016 16:05:30 -0700 Subject: [PATCH 329/871] Add a macro templatized hashtable. dynahash.c hash tables aren't quite fast enough for some use-cases. There are several reasons for lacking performance: - the use of chaining for collision handling makes them cache inefficient, that's especially an issue when the tables get bigger. - as the element sizes for dynahash are only determined at runtime, offset computations are somewhat expensive - hash and element comparisons are indirect function calls, causing unnecessary pipeline stalls - it's two level structure has some benefits (somewhat natural partitioning), but increases the number of indirections to fix several of these the hash tables have to be adjusted to the individual use-case at compile-time. C unfortunately doesn't provide a good way to do compile code generation (like e.g. c++'s templates for all their weaknesses do). Thus the somewhat ugly approach taken here is to allow for code generation using a macro-templatized header file, which generates functions and types based on a prefix and other parameters. Later patches use this infrastructure to use such hash tables for tidbitmap.c (bitmap scans) and execGrouping.c (hash aggregation, ...). In queries where these use up a large fraction of the time, this has been measured to lead to performance improvements of over 100%. There are other cases where this could be useful (e.g. catcache.c). The hash table design chosen is a variant of linear open-addressing. The biggest disadvantage of simple linear addressing schemes are highly variable lookup times due to clustering, and deletions leaving a lot of tombstones around. To address these issues a variant of "robin hood" hashing is employed. Robin hood hashing optimizes chaining lengths by moving elements close to their optimal bucket ("rich" elements), out of the way if a to-be-inserted element is further away from its optimal position (i.e. it's "poor"). While that can make insertions slower, the average lookup performance is a lot better, and higher fill factors can be used in a still performant manner. To avoid tombstones - which normally solve the issue that a deleted node's presence is relevant to determine whether a lookup needs to continue looking or is done - buckets following a deleted element are shifted backwards, unless they're empty or already at their optimal position. There's further possible improvements that can be made to this implementation. Amongst others: - Use distance as a termination criteria during searches. This is generally a good idea, but I've been able to see the overhead of distance calculations in some cases. - Consider combining the 'empty' status into the hashvalue, and enforce storing the hashvalue. That could, in some cases, increase memory density and remove a few instructions. - Experiment further with the, very conservatively choosen, fillfactor. - Make maximum size of hashtable configurable, to allow storing very very large tables. That'd require 64bit hash values to be more common than now, though. - some smaller memcpy calls could be optimized to copy larger chunks But since the new implementation is already considerably faster than dynahash it seem sensible to start using it. Reviewed-By: Tomas Vondra Discussion: <20160727004333.r3e2k2y6fvk2ntup@alap3.anarazel.de> --- src/include/lib/simplehash.h | 878 +++++++++++++++++++++++++++++++ src/tools/pgindent/typedefs.list | 3 + 2 files changed, 881 insertions(+) create mode 100644 src/include/lib/simplehash.h diff --git a/src/include/lib/simplehash.h b/src/include/lib/simplehash.h new file mode 100644 index 0000000000..bf89c41f1b --- /dev/null +++ b/src/include/lib/simplehash.h @@ -0,0 +1,878 @@ +/* + * simplehash.h + * + * Hash table implementation which will be specialized to user-defined + * types, by including this file to generate the required code. It's + * probably not worthwhile to do so for hash tables that aren't performance + * or space sensitive. + * + * Usage notes: + * + * To generate a hash-table and associated functions for a use case several + * macros have to be #define'ed before this file is included. Including + * the file #undef's all those, so a new hash table can be generated + * afterwards. + * The relevant parameters are: + * - SH_PREFIX - prefix for all symbol names generated. A prefix of 'foo' + * will result in hash table type 'foo_hash' and functions like + * 'foo_insert'/'foo_lookup' and so forth. + * - SH_ELEMENT_TYPE - type of the contained elements + * - SH_KEY_TYPE - type of the hashtable's key + * - SH_DECLARE - if defined function prototypes and type declarations are + * generated + * - SH_DEFINE - if defined function definitions are generated + * - SH_SCOPE - in which scope (e.g. extern, static inline) do function + * declarations reside + * The following parameters are only relevant when SH_DEFINE is defined: + * - SH_KEY - name of the element in SH_ELEMENT_TYPE containing the hash key + * - SH_EQUAL(table, a, b) - compare two table keys + * - SH_HASH_KEY(table, key) - generate hash for the key + * - SH_STORE_HASH - if defined the hash is stored in the elements + * - SH_GET_HASH(tb, a) - return the field to store the hash in + * + * For examples of usage look at simplehash.c (file local definition) and + * execnodes.h/execGrouping.c (exposed declaration, file local + * implementation). + * + * Hash table design: + * + * The hash table design chosen is a variant of linear open-addressing. The + * reason for doing so is that linear addressing is CPU cache & pipeline + * friendly. The biggest disadvantage of simple linear addressing schemes + * are highly variable lookup times due to clustering, and deletions + * leaving a lot of tombstones around. To address these issues a variant + * of "robin hood" hashing is employed. Robin hood hashing optimizes + * chaining lengths by moving elements close to their optimal bucket + * ("rich" elements), out of the way if a to-be-inserted element is further + * away from its optimal position (i.e. it's "poor"). While that can make + * insertions slower, the average lookup performance is a lot better, and + * higher fill factors can be used in a still performant manner. To avoid + * tombstones - which normally solve the issue that a deleted node's + * presence is relevant to determine whether a lookup needs to continue + * looking or is done - buckets following a deleted element are shifted + * backwards, unless they're empty or already at their optimal position. + */ + +/* helpers */ +#define SH_MAKE_PREFIX(a) CppConcat(a,_) +#define SH_MAKE_NAME(name) SH_MAKE_NAME_(SH_MAKE_PREFIX(SH_PREFIX),name) +#define SH_MAKE_NAME_(a,b) CppConcat(a,b) + +/* name macros for: */ + +/* type declarations */ +#define SH_TYPE SH_MAKE_NAME(hash) +#define SH_STATUS SH_MAKE_NAME(status) +#define SH_STATUS_EMPTY SH_MAKE_NAME(EMPTY) +#define SH_STATUS_IN_USE SH_MAKE_NAME(IN_USE) +#define SH_ITERATOR SH_MAKE_NAME(iterator) + +/* function declarations */ +#define SH_CREATE SH_MAKE_NAME(create) +#define SH_DESTROY SH_MAKE_NAME(destroy) +#define SH_INSERT SH_MAKE_NAME(insert) +#define SH_DELETE SH_MAKE_NAME(delete) +#define SH_LOOKUP SH_MAKE_NAME(lookup) +#define SH_GROW SH_MAKE_NAME(grow) +#define SH_START_ITERATE SH_MAKE_NAME(start_iterate) +#define SH_START_ITERATE_AT SH_MAKE_NAME(start_iterate_at) +#define SH_ITERATE SH_MAKE_NAME(iterate) +#define SH_STAT SH_MAKE_NAME(stat) + +/* internal helper functions (no externally visible prototypes) */ +#define SH_COMPUTE_PARAMETERS SH_MAKE_NAME(compute_parameters) +#define SH_NEXT SH_MAKE_NAME(next) +#define SH_PREV SH_MAKE_NAME(prev) +#define SH_DISTANCE_FROM_OPTIMAL SH_MAKE_NAME(distance) +#define SH_INITIAL_BUCKET SH_MAKE_NAME(initial_bucket) +#define SH_ENTRY_HASH SH_MAKE_NAME(entry_hash) + +/* generate forward declarations necessary to use the hash table */ +#ifdef SH_DECLARE + +/* type definitions */ +typedef struct SH_TYPE +{ + /* + * Size of data / bucket array, 64 bits to handle UINT32_MAX sized hash + * tables. Note that the maximum number of elements is lower + * (SH_MAX_FILLFACTOR) + */ + uint64 size; + + /* how many elements have valid contents */ + uint32 members; + + /* mask for bucket and size calculations, based on size */ + uint32 sizemask; + + /* boundary after which to grow hashtable */ + uint32 grow_threshold; + + /* hash buckets */ + SH_ELEMENT_TYPE *data; + + /* memory context to use for allocations */ + MemoryContext ctx; + + /* user defined data, useful for callbacks */ + void *private; +} SH_TYPE; + +typedef enum SH_STATUS +{ + SH_STATUS_EMPTY = 0x00, + SH_STATUS_IN_USE = 0x01 +} SH_STATUS; + +typedef struct SH_ITERATOR +{ + uint32 cur; /* current element */ + uint32 end; + bool done; /* iterator exhausted? */ +} SH_ITERATOR; + +/* externally visible function prototypes */ +SH_SCOPE SH_TYPE *SH_CREATE(MemoryContext ctx, uint32 nelements); +SH_SCOPE void SH_DESTROY(SH_TYPE *tb); +SH_SCOPE void SH_GROW(SH_TYPE *tb, uint32 newsize); +SH_SCOPE SH_ELEMENT_TYPE *SH_INSERT(SH_TYPE *tb, SH_KEY_TYPE key, bool *found); +SH_SCOPE SH_ELEMENT_TYPE *SH_LOOKUP(SH_TYPE *tb, SH_KEY_TYPE key); +SH_SCOPE bool SH_DELETE(SH_TYPE *tb, SH_KEY_TYPE key); +SH_SCOPE void SH_START_ITERATE(SH_TYPE *tb, SH_ITERATOR *iter); +SH_SCOPE void SH_START_ITERATE_AT(SH_TYPE *tb, SH_ITERATOR *iter, uint32 at); +SH_SCOPE SH_ELEMENT_TYPE *SH_ITERATE(SH_TYPE *tb, SH_ITERATOR *iter); +SH_SCOPE void SH_STAT(SH_TYPE *tb); + +#endif /* SH_DECLARE */ + + +/* generate implementation of the hash table */ +#ifdef SH_DEFINE + +#include "utils/memutils.h" + +/* conservative fillfactor for a robin hood table, might want to adjust */ +#define SH_FILLFACTOR (0.8) +/* increase fillfactor if we otherwise would error out */ +#define SH_MAX_FILLFACTOR (0.95) +/* max data array size,we allow up to PG_UINT32_MAX buckets, including 0 */ +#define SH_MAX_SIZE (((uint64) PG_UINT32_MAX) + 1) + +#ifdef SH_STORE_HASH +#define SH_COMPARE_KEYS(tb, ahash, akey, b) (ahash == SH_GET_HASH(tb, b) && SH_EQUAL(tb, b->SH_KEY, akey)) +#else +#define SH_COMPARE_KEYS(tb, ahash, akey, b) (SH_EQUAL(tb, b->SH_KEY, akey)) +#endif + +/* FIXME: can we move these to a central location? */ + +/* calculate ceil(log base 2) of num */ +static inline uint64 +sh_log2(uint64 num) +{ + int i; + uint64 limit; + + for (i = 0, limit = 1; limit < num; i++, limit <<= 1) + ; + return i; +} + +/* calculate first power of 2 >= num */ +static inline uint64 +sh_pow2(uint64 num) +{ + return ((uint64) 1) << sh_log2(num); +} + +/* + * Compute sizing parameters for hashtable. Called when creating and growing + * the hashtable. + */ +static inline void +SH_COMPUTE_PARAMETERS(SH_TYPE *tb, uint32 newsize) +{ + uint64 size; + + /* supporting zero sized hashes would complicate matters */ + size = Max(newsize, 2); + + /* round up size to the next power of 2, that's the bucketing works */ + size = sh_pow2(size); + Assert(size <= SH_MAX_SIZE); + + /* + * Verify allocation of ->data is possible on platform, without + * overflowing Size. + */ + if ((((uint64) sizeof(SH_ELEMENT_TYPE)) * size) >= MaxAllocHugeSize) + elog(ERROR, "hash table too large"); + + /* now set size */ + tb->size = size; + + if (tb->size == SH_MAX_SIZE) + tb->sizemask = 0; + else + tb->sizemask = tb->size - 1; + + /* + * Compute growth threshold here and after growing the table, to make + * computations during insert cheaper. + */ + if (tb->size == SH_MAX_SIZE) + tb->grow_threshold = ((double) tb->size) * SH_MAX_FILLFACTOR; + else + tb->grow_threshold = ((double) tb->size) * SH_FILLFACTOR; +} + +/* return the optimal bucket for the hash */ +static inline uint32 +SH_INITIAL_BUCKET(SH_TYPE *tb, uint32 hash) +{ + return hash & tb->sizemask; +} + +/* return next bucket after the current, handling wraparound */ +static inline uint32 +SH_NEXT(SH_TYPE *tb, uint32 curelem, uint32 startelem) +{ + curelem = (curelem + 1) & tb->sizemask; + + Assert(curelem != startelem); + + return curelem; +} + +/* return bucket before the current, handling wraparound */ +static inline uint32 +SH_PREV(SH_TYPE *tb, uint32 curelem, uint32 startelem) +{ + curelem = (curelem - 1) & tb->sizemask; + + Assert(curelem != startelem); + + return curelem; +} + +/* return distance between bucket and it's optimal position */ +static inline uint32 +SH_DISTANCE_FROM_OPTIMAL(SH_TYPE *tb, uint32 optimal, uint32 bucket) +{ + if (optimal <= bucket) + return bucket - optimal; + else + return (tb->size + bucket) - optimal; +} + +static inline uint32 +SH_ENTRY_HASH(SH_TYPE *tb, SH_ELEMENT_TYPE * entry) +{ +#ifdef SH_STORE_HASH + return SH_GET_HASH(tb, entry); +#else + return SH_HASH_KEY(tb, entry->SH_KEY); +#endif +} + +/* + * Create a hash table with enough space for `nelements` distinct members, + * allocating required memory in the passed-in context. + */ +SH_SCOPE SH_TYPE * +SH_CREATE(MemoryContext ctx, uint32 nelements) +{ + SH_TYPE *tb; + uint64 size; + + tb = MemoryContextAllocZero(ctx, sizeof(SH_TYPE)); + tb->ctx = ctx; + + /* increase nelements by fillfactor, want to store nelements elements */ + size = Min((double) SH_MAX_SIZE, ((double) nelements) / SH_FILLFACTOR); + + SH_COMPUTE_PARAMETERS(tb, size); + + tb->data = MemoryContextAllocExtended(tb->ctx, + sizeof(SH_ELEMENT_TYPE) * tb->size, + MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); + + return tb; +} + +/* destroy a previously created hash table */ +SH_SCOPE void +SH_DESTROY(SH_TYPE *tb) +{ + pfree(tb->data); + pfree(tb); +} + +/* + * Grow a hash table to at least `newsize` buckets. + * + * Usually this will automatically be called by insertions/deletions, when + * necessary. But resizing to the exact input size can be advantageous + * performance-wise, when known at some point. + */ +SH_SCOPE void +SH_GROW(SH_TYPE *tb, uint32 newsize) +{ + uint64 oldsize = tb->size; + SH_ELEMENT_TYPE *olddata = tb->data; + SH_ELEMENT_TYPE *newdata; + uint32 i; + uint32 startelem = 0; + uint32 copyelem; + + Assert(oldsize == sh_pow2(oldsize)); + Assert(oldsize != SH_MAX_SIZE); + Assert(oldsize < newsize); + + /* compute parameters for new table */ + SH_COMPUTE_PARAMETERS(tb, newsize); + + tb->data = MemoryContextAllocExtended( + tb->ctx, sizeof(SH_ELEMENT_TYPE) * tb->size, + MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); + + newdata = tb->data; + + /* + * Copy entries from the old data to newdata. We theoretically could use + * SH_INSERT here, to avoid code duplication, but that's more general than + * we need. We neither want tb->members increased, nor do we need to do + * deal with deleted elements, nor do we need to compare keys. So a + * special-cased implementation is lot faster. As resizing can be time + * consuming and frequent, that's worthwile to optimize. + * + * To be able to simply move entries over, we have to start not at the + * first bucket (i.e olddata[0]), but find the first bucket that's either + * empty, or is occupied by an entry at it's optimal position. Such a + * bucket has to exist in any table with a load factor under 1, as not all + * buckets are occupied, i.e. there always has to be an empty bucket. By + * starting at such a bucket we can move the entries to the larger table, + * without having to deal with conflicts. + */ + + /* search for the first element in the hash that's not wrapped around */ + for (i = 0; i < oldsize; i++) + { + SH_ELEMENT_TYPE *oldentry = &olddata[i]; + uint32 hash; + uint32 optimal; + + if (oldentry->status != SH_STATUS_IN_USE) + { + startelem = i; + break; + } + + hash = SH_ENTRY_HASH(tb, oldentry); + optimal = SH_INITIAL_BUCKET(tb, hash); + + if (optimal == i) + { + startelem = i; + break; + } + } + + /* and copy all elements in the old table */ + copyelem = startelem; + for (i = 0; i < oldsize; i++) + { + SH_ELEMENT_TYPE *oldentry = &olddata[copyelem]; + + if (oldentry->status == SH_STATUS_IN_USE) + { + uint32 hash; + uint32 startelem; + uint32 curelem; + SH_ELEMENT_TYPE *newentry; + + hash = SH_ENTRY_HASH(tb, oldentry); + startelem = SH_INITIAL_BUCKET(tb, hash); + curelem = startelem; + + /* find empty element to put data into */ + while (true) + { + newentry = &newdata[curelem]; + + if (newentry->status == SH_STATUS_EMPTY) + { + break; + } + + curelem = SH_NEXT(tb, curelem, startelem); + } + + /* copy entry to new slot */ + memcpy(newentry, oldentry, sizeof(SH_ELEMENT_TYPE)); + } + + /* can't use SH_NEXT here, would use new size */ + copyelem++; + if (copyelem >= oldsize) + { + copyelem = 0; + } + } + + pfree(olddata); +} + +/* + * Insert the key key into the hash-table, set *found to true if the key + * already exists, false otherwise. Returns the hash-table entry in either + * case. + */ +SH_SCOPE SH_ELEMENT_TYPE * +SH_INSERT(SH_TYPE *tb, SH_KEY_TYPE key, bool *found) +{ + uint32 hash = SH_HASH_KEY(tb, key); + uint32 startelem; + uint32 curelem; + SH_ELEMENT_TYPE *data; + uint32 insertdist = 0; + + /* + * We do the grow check even if the key is actually present, to avoid + * doing the check inside the loop. This also lets us avoid having to + * re-find our position in the hashtable after resizing. + */ + if (unlikely(tb->members >= tb->grow_threshold)) + { + if (tb->size == SH_MAX_SIZE) + { + elog(ERROR, "hash table size exceeded"); + } + + /* + * When optimizing, it can be very useful to print these out. + */ + /* SH_STAT(tb); */ + SH_GROW(tb, tb->size * 2); + /* SH_STAT(tb); */ + } + + /* perform insert, start bucket search at optimal location */ + data = tb->data; + startelem = SH_INITIAL_BUCKET(tb, hash); + curelem = startelem; + while (true) + { + uint32 curdist; + uint32 curhash; + uint32 curoptimal; + SH_ELEMENT_TYPE *entry = &data[curelem]; + + /* any empty bucket can directly be used */ + if (entry->status == SH_STATUS_EMPTY) + { + tb->members++; + entry->SH_KEY = key; +#ifdef SH_STORE_HASH + SH_GET_HASH(tb, entry) = hash; +#endif + entry->status = SH_STATUS_IN_USE; + *found = false; + return entry; + } + + /* + * If the bucket is not empty, we either found a match (in which case + * we're done), or we have to decide whether to skip over or move the + * colliding entry. When the the colliding elements distance to it's + * optimal position is smaller than the to-be-inserted entry's, we + * shift the colliding entry (and it's followers) forward by one. + */ + + if (SH_COMPARE_KEYS(tb, hash, key, entry)) + { + Assert(entry->status == SH_STATUS_IN_USE); + *found = true; + return entry; + } + + curhash = SH_ENTRY_HASH(tb, entry); + curoptimal = SH_INITIAL_BUCKET(tb, curhash); + curdist = SH_DISTANCE_FROM_OPTIMAL(tb, curoptimal, curelem); + + if (insertdist > curdist) + { + SH_ELEMENT_TYPE *lastentry = entry; + uint32 emptyelem = curelem; + uint32 moveelem; + + /* find next empty bucket */ + while (true) + { + SH_ELEMENT_TYPE *emptyentry; + + emptyelem = SH_NEXT(tb, emptyelem, startelem); + emptyentry = &data[emptyelem]; + + if (emptyentry->status == SH_STATUS_EMPTY) + { + lastentry = emptyentry; + break; + } + } + + /* shift forward, starting at last occupied element */ + + /* + * TODO: This could be optimized to be one memcpy in may cases, + * excepting wrapping around at the end of ->data. Hasn't shown up + * in profiles so far though. + */ + moveelem = emptyelem; + while (moveelem != curelem) + { + SH_ELEMENT_TYPE *moveentry; + + moveelem = SH_PREV(tb, moveelem, startelem); + moveentry = &data[moveelem]; + + memcpy(lastentry, moveentry, sizeof(SH_ELEMENT_TYPE)); + lastentry = moveentry; + } + + /* and fill the now empty spot */ + tb->members++; + + entry->SH_KEY = key; +#ifdef SH_STORE_HASH + SH_GET_HASH(tb, entry) = hash; +#endif + entry->status = SH_STATUS_IN_USE; + *found = false; + return entry; + } + + curelem = SH_NEXT(tb, curelem, startelem); + insertdist++; + } +} + +/* + * Lookup up entry in hash table. Returns NULL if key not present. + */ +SH_SCOPE SH_ELEMENT_TYPE * +SH_LOOKUP(SH_TYPE *tb, SH_KEY_TYPE key) +{ + uint32 hash = SH_HASH_KEY(tb, key); + const uint32 startelem = SH_INITIAL_BUCKET(tb, hash); + uint32 curelem = startelem; + + while (true) + { + SH_ELEMENT_TYPE *entry = &tb->data[curelem]; + + if (entry->status == SH_STATUS_EMPTY) + { + return NULL; + } + + Assert(entry->status == SH_STATUS_IN_USE); + + if (SH_COMPARE_KEYS(tb, hash, key, entry)) + return entry; + + /* + * TODO: we could stop search based on distance. If the current + * buckets's distance-from-optimal is smaller than what we've skipped + * already, the entry doesn't exist. Probably only do so if + * SH_STORE_HASH is defined, to avoid re-computing hashes? + */ + + curelem = SH_NEXT(tb, curelem, startelem); + } +} + +/* + * Delete entry from hash table. Returns whether to-be-deleted key was + * present. + */ +SH_SCOPE bool +SH_DELETE(SH_TYPE *tb, SH_KEY_TYPE key) +{ + uint32 hash = SH_HASH_KEY(tb, key); + uint32 startelem = SH_INITIAL_BUCKET(tb, hash); + uint32 curelem = startelem; + + while (true) + { + SH_ELEMENT_TYPE *entry = &tb->data[curelem]; + + if (entry->status == SH_STATUS_EMPTY) + return false; + + if (entry->status == SH_STATUS_IN_USE && + SH_COMPARE_KEYS(tb, hash, key, entry)) + { + SH_ELEMENT_TYPE *lastentry = entry; + + tb->members--; + + /* + * Backward shift following elements till either an empty element + * or an element at its optimal position is encounterered. + * + * While that sounds expensive, the average chain length is short, + * and deletions would otherwise require toombstones. + */ + while (true) + { + SH_ELEMENT_TYPE *curentry; + uint32 curhash; + uint32 curoptimal; + + curelem = SH_NEXT(tb, curelem, startelem); + curentry = &tb->data[curelem]; + + if (curentry->status != SH_STATUS_IN_USE) + { + lastentry->status = SH_STATUS_EMPTY; + break; + } + + curhash = SH_ENTRY_HASH(tb, curentry); + curoptimal = SH_INITIAL_BUCKET(tb, curhash); + + /* current is at optimal position, done */ + if (curoptimal == curelem) + { + lastentry->status = SH_STATUS_EMPTY; + break; + } + + /* shift */ + memcpy(lastentry, curentry, sizeof(SH_ELEMENT_TYPE)); + + lastentry = curentry; + } + + return true; + } + + /* TODO: return false; if distance too big */ + + curelem = SH_NEXT(tb, curelem, startelem); + } +} + +/* + * Initialize iterator. + */ +SH_SCOPE void +SH_START_ITERATE(SH_TYPE *tb, SH_ITERATOR *iter) +{ + int i; + uint64 startelem = PG_UINT64_MAX; + + /* + * Search for the first empty element. As deletions during iterations are + * supported, we want to start/end at an element that cannot be affected + * by elements being shifted. + */ + for (i = 0; i < tb->size; i++) + { + SH_ELEMENT_TYPE *entry = &tb->data[i]; + + if (entry->status != SH_STATUS_IN_USE) + { + startelem = i; + break; + } + } + + Assert(startelem < SH_MAX_SIZE); + + /* + * Iterate backwards, that allows the current element to be deleted, even + * if there are backward shifts + */ + iter->cur = startelem; + iter->end = iter->cur; + iter->done = false; +} + +/* + * Initialize iterator to a specific bucket. That's really only useful for + * cases where callers are partially iterating over the hashspace, and that + * iteration deletes and inserts elements based on visited entries. Doing that + * repeatedly could lead to an unbalanced keyspace when always starting at the + * same position. + */ +SH_SCOPE void +SH_START_ITERATE_AT(SH_TYPE *tb, SH_ITERATOR *iter, uint32 at) +{ + /* + * Iterate backwards, that allows the current element to be deleted, even + * if there are backward shifts. + */ + iter->cur = at & tb->sizemask; /* ensure at is within a valid range */ + iter->end = iter->cur; + iter->done = false; +} + +/* + * Iterate over all entries in the hash-table. Return the next occupied entry, + * or NULL if done. + * + * During iteration the current entry in the hash table may be deleted, + * without leading to elements being skipped or returned twice. Additionally + * the rest of the table may be modified (i.e. there can be insertions or + * deletions), but if so, there's neither a guarantee that all nodes are + * visited at least once, nor a guarantee that a node is visited at most once. + */ +SH_SCOPE SH_ELEMENT_TYPE * +SH_ITERATE(SH_TYPE *tb, SH_ITERATOR *iter) +{ + while (!iter->done) + { + SH_ELEMENT_TYPE *elem; + + elem = &tb->data[iter->cur]; + + /* next element in backward direction */ + iter->cur = (iter->cur - 1) & tb->sizemask; + + if ((iter->cur & tb->sizemask) == (iter->end & tb->sizemask)) + iter->done = true; + if (elem->status == SH_STATUS_IN_USE) + { + return elem; + } + } + + return NULL; +} + +/* + * Report some statistics about the state of the hashtable. For + * debugging/profiling purposes only. + */ +SH_SCOPE void +SH_STAT(SH_TYPE *tb) +{ + uint32 max_chain_length = 0; + uint32 total_chain_length = 0; + double avg_chain_length; + double fillfactor; + uint32 i; + + uint32 *collisions = palloc0(tb->size * sizeof(uint32)); + uint32 total_collisions = 0; + uint32 max_collisions = 0; + double avg_collisions; + + for (i = 0; i < tb->size; i++) + { + uint32 hash; + uint32 optimal; + uint32 dist; + SH_ELEMENT_TYPE *elem; + + elem = &tb->data[i]; + + if (elem->status != SH_STATUS_IN_USE) + continue; + + hash = SH_ENTRY_HASH(tb, elem); + optimal = SH_INITIAL_BUCKET(tb, hash); + dist = SH_DISTANCE_FROM_OPTIMAL(tb, optimal, i); + + if (dist > max_chain_length) + max_chain_length = dist; + total_chain_length += dist; + + collisions[optimal]++; + } + + for (i = 0; i < tb->size; i++) + { + uint32 curcoll = collisions[i]; + + if (curcoll == 0) + continue; + + /* single contained element is not a collision */ + curcoll--; + total_collisions += curcoll; + if (curcoll > max_collisions) + max_collisions = curcoll; + } + + if (tb->members > 0) + { + fillfactor = tb->members / ((double) tb->size); + avg_chain_length = ((double) total_chain_length) / tb->members; + avg_collisions = ((double) total_collisions) / tb->members; + } + else + { + fillfactor = 0; + avg_chain_length = 0; + avg_collisions = 0; + } + + elog(LOG, "size: " UINT64_FORMAT ", members: %u, filled: %f, total chain: %u, max chain: %u, avg chain: %f, total_collisions: %u, max_collisions: %i, avg_collisions: %f", + tb->size, tb->members, fillfactor, total_chain_length, max_chain_length, avg_chain_length, + total_collisions, max_collisions, avg_collisions); +} + +#endif /* SH_DEFINE */ + + +/* undefine external paramters, so next hash table can be defined */ +#undef SH_PREFIX +#undef SH_KEY_TYPE +#undef SH_KEY +#undef SH_ELEMENT_TYPE +#undef SH_HASH_KEY +#undef SH_SCOPE +#undef SH_DECLARE +#undef SH_DEFINE +#undef SH_GET_HASH +#undef SH_STORE_HASH + +/* undefine locally declared macros */ +#undef SH_MAKE_PREFIX +#undef SH_MAKE_NAME +#undef SH_MAKE_NAME_ +#undef SH_FILLFACTOR +#undef SH_MAX_FILLFACTOR +#undef SH_MAX_SIZE + +/* types */ +#undef SH_TYPE +#undef SH_STATUS +#undef SH_STATUS_EMPTY +#undef SH_STATUS_IN_USE +#undef SH_ITERTOR + +/* external function names */ +#undef SH_CREATE +#undef SH_DESTROY +#undef SH_INSERT +#undef SH_DELETE +#undef SH_LOOKUP +#undef SH_GROW +#undef SH_START_ITERATE +#undef SH_START_ITERATE_AT +#undef SH_ITERATE +#undef SH_STAT + +/* internal function names */ +#undef SH_COMPUTE_PARAMETERS +#undef SH_COMPARE_KEYS +#undef SH_INITIAL_BUCKET +#undef SH_NEXT +#undef SH_PREV +#undef SH_DISTANCE_FROM_OPTIMAL +#undef SH_ENTRY_HASH diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 932be2f665..e61ebbfe2a 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1797,6 +1797,9 @@ SERIALIZABLEXIDTAG SERVICE_STATUS SERVICE_STATUS_HANDLE SERVICE_TABLE_ENTRY +SH_TYPE +SH_ITERATOR +SH_STATUS SHA1_CTX SHA224_CTX SHA256_CTX From 75ae538bc3168bf44475240d4e0487ee2f3bb376 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Fri, 14 Oct 2016 16:05:30 -0700 Subject: [PATCH 330/871] Use more efficient hashtable for tidbitmap.c to speed up bitmap scans. Use the new simplehash.h to speed up tidbitmap.c uses. For bitmap scan heavy queries speedups of over 100% have been measured. Both lossy and exact scans benefit, but the wins are bigger for mostly exact scans. The conversion is mostly trivial, except that tbm_lossify() now restarts lossifying at the point it previously stopped. Otherwise the hash table becomes unbalanced because the scan in done in hash-order, leaving the end of the hashtable more densely filled then the beginning. That caused performance issues with dynahash as well, but due to the open chaining they were less pronounced than with the linear adressing from simplehash.h. Reviewed-By: Tomas Vondra Discussion: <20160727004333.r3e2k2y6fvk2ntup@alap3.anarazel.de> --- src/backend/nodes/tidbitmap.c | 169 +++++++++++++++++++--------------- 1 file changed, 97 insertions(+), 72 deletions(-) diff --git a/src/backend/nodes/tidbitmap.c b/src/backend/nodes/tidbitmap.c index dfeb7d5c63..826feada1f 100644 --- a/src/backend/nodes/tidbitmap.c +++ b/src/backend/nodes/tidbitmap.c @@ -43,7 +43,6 @@ #include "access/htup_details.h" #include "nodes/bitmapset.h" #include "nodes/tidbitmap.h" -#include "utils/hsearch.h" /* * The maximum number of tuples per page is not large (typically 256 with @@ -61,12 +60,12 @@ * for that page in the page table. * * We actually store both exact pages and lossy chunks in the same hash - * table, using identical data structures. (This is because dynahash.c's - * memory management doesn't allow space to be transferred easily from one - * hashtable to another.) Therefore it's best if PAGES_PER_CHUNK is the - * same as MAX_TUPLES_PER_PAGE, or at least not too different. But we - * also want PAGES_PER_CHUNK to be a power of 2 to avoid expensive integer - * remainder operations. So, define it like this: + * table, using identical data structures. (This is because the memory + * management for hashtables doesn't easily/efficiently allow space to be + * transferred easily from one hashtable to another.) Therefore it's best + * if PAGES_PER_CHUNK is the same as MAX_TUPLES_PER_PAGE, or at least not + * too different. But we also want PAGES_PER_CHUNK to be a power of 2 to + * avoid expensive integer remainder operations. So, define it like this: */ #define PAGES_PER_CHUNK (BLCKSZ / 32) @@ -97,21 +96,22 @@ typedef struct PagetableEntry { BlockNumber blockno; /* page number (hashtable key) */ + char status; /* hash entry status */ bool ischunk; /* T = lossy storage, F = exact */ bool recheck; /* should the tuples be rechecked? */ bitmapword words[Max(WORDS_PER_PAGE, WORDS_PER_CHUNK)]; } PagetableEntry; /* - * dynahash.c is optimized for relatively large, long-lived hash tables. - * This is not ideal for TIDBitMap, particularly when we are using a bitmap - * scan on the inside of a nestloop join: a bitmap may well live only long - * enough to accumulate one entry in such cases. We therefore avoid creating - * an actual hashtable until we need two pagetable entries. When just one - * pagetable entry is needed, we store it in a fixed field of TIDBitMap. - * (NOTE: we don't get rid of the hashtable if the bitmap later shrinks down - * to zero or one page again. So, status can be TBM_HASH even when nentries - * is zero or one.) + * We want to avoid the overhead of creating the hashtable, which is + * comparatively large, when not necessary. Particularly when we are using a + * bitmap scan on the inside of a nestloop join: a bitmap may well live only + * long enough to accumulate one entry in such cases. We therefore avoid + * creating an actual hashtable until we need two pagetable entries. When + * just one pagetable entry is needed, we store it in a fixed field of + * TIDBitMap. (NOTE: we don't get rid of the hashtable if the bitmap later + * shrinks down to zero or one page again. So, status can be TBM_HASH even + * when nentries is zero or one.) */ typedef enum { @@ -128,12 +128,13 @@ struct TIDBitmap NodeTag type; /* to make it a valid Node */ MemoryContext mcxt; /* memory context containing me */ TBMStatus status; /* see codes above */ - HTAB *pagetable; /* hash table of PagetableEntry's */ + struct pagetable_hash *pagetable; /* hash table of PagetableEntry's */ int nentries; /* number of entries in pagetable */ int maxentries; /* limit on same to meet maxbytes */ int npages; /* number of exact entries in pagetable */ int nchunks; /* number of lossy entries in pagetable */ bool iterating; /* tbm_begin_iterate called? */ + uint32 lossify_start; /* offset to start lossifying hashtable at */ PagetableEntry entry1; /* used when status == TBM_ONE_PAGE */ /* these are valid when iterating is true: */ PagetableEntry **spages; /* sorted exact-page list, or NULL */ @@ -168,6 +169,35 @@ static void tbm_mark_page_lossy(TIDBitmap *tbm, BlockNumber pageno); static void tbm_lossify(TIDBitmap *tbm); static int tbm_comparator(const void *left, const void *right); +/* + * Simple inline murmur hash implementation for the exact width required, for + * performance. + */ +static inline uint32 +hash_blockno(BlockNumber b) +{ + uint32 h = b; + + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} + +/* define hashtable mapping block numbers to PagetableEntry's */ +#define SH_PREFIX pagetable +#define SH_ELEMENT_TYPE PagetableEntry +#define SH_KEY_TYPE BlockNumber +#define SH_KEY blockno +#define SH_HASH_KEY(tb, key) hash_blockno(key) +#define SH_EQUAL(tb, a, b) a == b +#define SH_SCOPE static inline +#define SH_DEFINE +#define SH_DECLARE +#include "lib/simplehash.h" + /* * tbm_create - create an initially-empty bitmap @@ -190,17 +220,16 @@ tbm_create(long maxbytes) /* * Estimate number of hashtable entries we can have within maxbytes. This - * estimates the hash overhead at MAXALIGN(sizeof(HASHELEMENT)) plus a - * pointer per hash entry, which is crude but good enough for our purpose. - * Also count an extra Pointer per entry for the arrays created during - * iteration readout. + * estimates the hash cost as sizeof(PagetableEntry), which is good enough + * for our purpose. Also count an extra Pointer per entry for the arrays + * created during iteration readout. */ nbuckets = maxbytes / - (MAXALIGN(sizeof(HASHELEMENT)) + MAXALIGN(sizeof(PagetableEntry)) - + sizeof(Pointer) + sizeof(Pointer)); + (sizeof(PagetableEntry) + sizeof(Pointer) + sizeof(Pointer)); nbuckets = Min(nbuckets, INT_MAX - 1); /* safety limit */ nbuckets = Max(nbuckets, 16); /* sanity limit */ tbm->maxentries = (int) nbuckets; + tbm->lossify_start = 0; return tbm; } @@ -212,32 +241,25 @@ tbm_create(long maxbytes) static void tbm_create_pagetable(TIDBitmap *tbm) { - HASHCTL hash_ctl; - Assert(tbm->status != TBM_HASH); Assert(tbm->pagetable == NULL); - /* Create the hashtable proper */ - MemSet(&hash_ctl, 0, sizeof(hash_ctl)); - hash_ctl.keysize = sizeof(BlockNumber); - hash_ctl.entrysize = sizeof(PagetableEntry); - hash_ctl.hcxt = tbm->mcxt; - tbm->pagetable = hash_create("TIDBitmap", - 128, /* start small and extend */ - &hash_ctl, - HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + tbm->pagetable = pagetable_create(tbm->mcxt, 128); /* If entry1 is valid, push it into the hashtable */ if (tbm->status == TBM_ONE_PAGE) { PagetableEntry *page; bool found; + char oldstatus; - page = (PagetableEntry *) hash_search(tbm->pagetable, - (void *) &tbm->entry1.blockno, - HASH_ENTER, &found); + page = pagetable_insert(tbm->pagetable, + tbm->entry1.blockno, + &found); Assert(!found); + oldstatus = page->status; memcpy(page, &tbm->entry1, sizeof(PagetableEntry)); + page->status = oldstatus; } tbm->status = TBM_HASH; @@ -250,7 +272,7 @@ void tbm_free(TIDBitmap *tbm) { if (tbm->pagetable) - hash_destroy(tbm->pagetable); + pagetable_destroy(tbm->pagetable); if (tbm->spages) pfree(tbm->spages); if (tbm->schunks) @@ -357,12 +379,12 @@ tbm_union(TIDBitmap *a, const TIDBitmap *b) tbm_union_page(a, &b->entry1); else { - HASH_SEQ_STATUS status; + pagetable_iterator i; PagetableEntry *bpage; Assert(b->status == TBM_HASH); - hash_seq_init(&status, b->pagetable); - while ((bpage = (PagetableEntry *) hash_seq_search(&status)) != NULL) + pagetable_start_iterate(b->pagetable, &i); + while ((bpage = pagetable_iterate(b->pagetable, &i)) != NULL) tbm_union_page(a, bpage); } } @@ -449,12 +471,12 @@ tbm_intersect(TIDBitmap *a, const TIDBitmap *b) } else { - HASH_SEQ_STATUS status; + pagetable_iterator i; PagetableEntry *apage; Assert(a->status == TBM_HASH); - hash_seq_init(&status, a->pagetable); - while ((apage = (PagetableEntry *) hash_seq_search(&status)) != NULL) + pagetable_start_iterate(a->pagetable, &i); + while ((apage = pagetable_iterate(a->pagetable, &i)) != NULL) { if (tbm_intersect_page(a, apage, b)) { @@ -464,9 +486,7 @@ tbm_intersect(TIDBitmap *a, const TIDBitmap *b) else a->npages--; a->nentries--; - if (hash_search(a->pagetable, - (void *) &apage->blockno, - HASH_REMOVE, NULL) == NULL) + if (!pagetable_delete(a->pagetable, apage->blockno)) elog(ERROR, "hash table corrupted"); } } @@ -606,7 +626,7 @@ tbm_begin_iterate(TIDBitmap *tbm) */ if (tbm->status == TBM_HASH && !tbm->iterating) { - HASH_SEQ_STATUS status; + pagetable_iterator i; PagetableEntry *page; int npages; int nchunks; @@ -620,9 +640,9 @@ tbm_begin_iterate(TIDBitmap *tbm) MemoryContextAlloc(tbm->mcxt, tbm->nchunks * sizeof(PagetableEntry *)); - hash_seq_init(&status, tbm->pagetable); npages = nchunks = 0; - while ((page = (PagetableEntry *) hash_seq_search(&status)) != NULL) + pagetable_start_iterate(tbm->pagetable, &i); + while ((page = pagetable_iterate(tbm->pagetable, &i)) != NULL) { if (page->ischunk) tbm->schunks[nchunks++] = page; @@ -791,9 +811,7 @@ tbm_find_pageentry(const TIDBitmap *tbm, BlockNumber pageno) return page; } - page = (PagetableEntry *) hash_search(tbm->pagetable, - (void *) &pageno, - HASH_FIND, NULL); + page = pagetable_lookup(tbm->pagetable, pageno); if (page == NULL) return NULL; if (page->ischunk) @@ -834,15 +852,16 @@ tbm_get_pageentry(TIDBitmap *tbm, BlockNumber pageno) } /* Look up or create an entry */ - page = (PagetableEntry *) hash_search(tbm->pagetable, - (void *) &pageno, - HASH_ENTER, &found); + page = pagetable_insert(tbm->pagetable, pageno, &found); } /* Initialize it if not present before */ if (!found) { + char oldstatus = page->status; + MemSet(page, 0, sizeof(PagetableEntry)); + page->status = oldstatus; page->blockno = pageno; /* must count it too */ tbm->nentries++; @@ -869,9 +888,9 @@ tbm_page_is_lossy(const TIDBitmap *tbm, BlockNumber pageno) bitno = pageno % PAGES_PER_CHUNK; chunk_pageno = pageno - bitno; - page = (PagetableEntry *) hash_search(tbm->pagetable, - (void *) &chunk_pageno, - HASH_FIND, NULL); + + page = pagetable_lookup(tbm->pagetable, chunk_pageno); + if (page != NULL && page->ischunk) { int wordnum = WORDNUM(bitno); @@ -912,9 +931,7 @@ tbm_mark_page_lossy(TIDBitmap *tbm, BlockNumber pageno) */ if (bitno != 0) { - if (hash_search(tbm->pagetable, - (void *) &pageno, - HASH_REMOVE, NULL) != NULL) + if (pagetable_delete(tbm->pagetable, pageno)) { /* It was present, so adjust counts */ tbm->nentries--; @@ -923,14 +940,15 @@ tbm_mark_page_lossy(TIDBitmap *tbm, BlockNumber pageno) } /* Look up or create entry for chunk-header page */ - page = (PagetableEntry *) hash_search(tbm->pagetable, - (void *) &chunk_pageno, - HASH_ENTER, &found); + page = pagetable_insert(tbm->pagetable, chunk_pageno, &found); /* Initialize it if not present before */ if (!found) { + char oldstatus = page->status; + MemSet(page, 0, sizeof(PagetableEntry)); + page->status = oldstatus; page->blockno = chunk_pageno; page->ischunk = true; /* must count it too */ @@ -939,8 +957,11 @@ tbm_mark_page_lossy(TIDBitmap *tbm, BlockNumber pageno) } else if (!page->ischunk) { + char oldstatus = page->status; + /* chunk header page was formerly non-lossy, make it lossy */ MemSet(page, 0, sizeof(PagetableEntry)); + page->status = oldstatus; page->blockno = chunk_pageno; page->ischunk = true; /* we assume it had some tuple bit(s) set, so mark it lossy */ @@ -962,7 +983,7 @@ tbm_mark_page_lossy(TIDBitmap *tbm, BlockNumber pageno) static void tbm_lossify(TIDBitmap *tbm) { - HASH_SEQ_STATUS status; + pagetable_iterator i; PagetableEntry *page; /* @@ -977,8 +998,8 @@ tbm_lossify(TIDBitmap *tbm) Assert(!tbm->iterating); Assert(tbm->status == TBM_HASH); - hash_seq_init(&status, tbm->pagetable); - while ((page = (PagetableEntry *) hash_seq_search(&status)) != NULL) + pagetable_start_iterate_at(tbm->pagetable, &i, tbm->lossify_start); + while ((page = pagetable_iterate(tbm->pagetable, &i)) != NULL) { if (page->ischunk) continue; /* already a chunk header */ @@ -995,15 +1016,19 @@ tbm_lossify(TIDBitmap *tbm) if (tbm->nentries <= tbm->maxentries / 2) { - /* we have done enough */ - hash_seq_term(&status); + /* + * We have made enough room. Remember where to start lossifying + * next round, so we evenly iterate over the hashtable. + */ + tbm->lossify_start = i.cur; break; } /* * Note: tbm_mark_page_lossy may have inserted a lossy chunk into the - * hashtable. We can continue the same seq_search scan since we do - * not care whether we visit lossy chunks or not. + * hashtable and may have deleted the non-lossy chunk. We can + * continue the same hash table scan, since failure to visit one + * element or visiting the newly inserted element, isn't fatal. */ } From 5dfc198146b49ce7ecc8a1fc9d5e171fb75f6ba5 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Fri, 14 Oct 2016 17:22:51 -0700 Subject: [PATCH 331/871] Use more efficient hashtable for execGrouping.c to speed up hash aggregation. The more efficient hashtable speeds up hash-aggregations with more than a few hundred groups significantly. Improvements of over 120% have been measured. Due to the the different hash table queries that not fully determined (e.g. GROUP BY without ORDER BY) may change their result order. The conversion is largely straight-forward, except that, due to the static element types of simplehash.h type hashes, the additional data some users store in elements (e.g. the per-group working data for hash aggregaters) is now stored in TupleHashEntryData->additional. The meaning of BuildTupleHashTable's entrysize (renamed to additionalsize) has been changed to only be about the additionally stored size. That size is only used for the initial sizing of the hash-table. Reviewed-By: Tomas Vondra Discussion: <20160727004333.r3e2k2y6fvk2ntup@alap3.anarazel.de> --- src/backend/executor/execGrouping.c | 155 +++++++++------------- src/backend/executor/nodeAgg.c | 64 ++++----- src/backend/executor/nodeRecursiveunion.c | 17 +-- src/backend/executor/nodeSetOp.c | 46 +++---- src/backend/executor/nodeSubplan.c | 6 +- src/backend/optimizer/plan/planner.c | 6 + src/include/executor/executor.h | 2 +- src/include/nodes/execnodes.h | 32 +++-- src/tools/pgindent/typedefs.list | 2 + 9 files changed, 144 insertions(+), 186 deletions(-) diff --git a/src/backend/executor/execGrouping.c b/src/backend/executor/execGrouping.c index 808275a094..5a4e736476 100644 --- a/src/backend/executor/execGrouping.c +++ b/src/backend/executor/execGrouping.c @@ -23,12 +23,25 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" +static uint32 TupleHashTableHash(struct tuplehash_hash *tb, const MinimalTuple tuple); +static int TupleHashTableMatch(struct tuplehash_hash *tb, const MinimalTuple tuple1, const MinimalTuple tuple2); -static TupleHashTable CurTupleHashTable = NULL; - -static uint32 TupleHashTableHash(const void *key, Size keysize); -static int TupleHashTableMatch(const void *key1, const void *key2, - Size keysize); +/* + * Define parameters for tuple hash table code generation. The interface is + * *also* declared in execnodes.h (to generate the types, which are externally + * visible). + */ +#define SH_PREFIX tuplehash +#define SH_ELEMENT_TYPE TupleHashEntryData +#define SH_KEY_TYPE MinimalTuple +#define SH_KEY firstTuple +#define SH_HASH_KEY(tb, key) TupleHashTableHash(tb, key) +#define SH_EQUAL(tb, a, b) TupleHashTableMatch(tb, a, b) == 0 +#define SH_SCOPE extern +#define SH_STORE_HASH +#define SH_GET_HASH(tb, a) a->hash +#define SH_DEFINE +#include "lib/simplehash.h" /***************************************************************************** @@ -260,7 +273,7 @@ execTuplesHashPrepare(int numCols, * eqfunctions: equality comparison functions to use * hashfunctions: datatype-specific hashing functions to use * nbuckets: initial estimate of hashtable size - * entrysize: size of each entry (at least sizeof(TupleHashEntryData)) + * additionalsize: size of data stored in ->additional * tablecxt: memory context in which to store table and table entries * tempcxt: short-lived context for evaluation hash and comparison functions * @@ -275,20 +288,19 @@ TupleHashTable BuildTupleHashTable(int numCols, AttrNumber *keyColIdx, FmgrInfo *eqfunctions, FmgrInfo *hashfunctions, - long nbuckets, Size entrysize, + long nbuckets, Size additionalsize, MemoryContext tablecxt, MemoryContext tempcxt) { TupleHashTable hashtable; - HASHCTL hash_ctl; + Size entrysize = sizeof(TupleHashEntryData) + additionalsize; Assert(nbuckets > 0); - Assert(entrysize >= sizeof(TupleHashEntryData)); /* Limit initial table size request to not more than work_mem */ nbuckets = Min(nbuckets, (long) ((work_mem * 1024L) / entrysize)); - hashtable = (TupleHashTable) MemoryContextAlloc(tablecxt, - sizeof(TupleHashTableData)); + hashtable = (TupleHashTable) + MemoryContextAlloc(tablecxt, sizeof(TupleHashTableData)); hashtable->numCols = numCols; hashtable->keyColIdx = keyColIdx; @@ -302,15 +314,8 @@ BuildTupleHashTable(int numCols, AttrNumber *keyColIdx, hashtable->in_hash_funcs = NULL; hashtable->cur_eq_funcs = NULL; - MemSet(&hash_ctl, 0, sizeof(hash_ctl)); - hash_ctl.keysize = sizeof(TupleHashEntryData); - hash_ctl.entrysize = entrysize; - hash_ctl.hash = TupleHashTableHash; - hash_ctl.match = TupleHashTableMatch; - hash_ctl.hcxt = tablecxt; - hashtable->hashtab = hash_create("TupleHashTable", nbuckets, - &hash_ctl, - HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT); + hashtable->hashtab = tuplehash_create(tablecxt, nbuckets); + hashtable->hashtab->private = hashtable; return hashtable; } @@ -324,18 +329,17 @@ BuildTupleHashTable(int numCols, AttrNumber *keyColIdx, * * If isnew isn't NULL, then a new entry is created if no existing entry * matches. On return, *isnew is true if the entry is newly created, - * false if it existed already. Any extra space in a new entry has been - * zeroed. + * false if it existed already. ->additional_data in the new entry has + * been zeroed. */ TupleHashEntry LookupTupleHashEntry(TupleHashTable hashtable, TupleTableSlot *slot, bool *isnew) { - TupleHashEntry entry; + TupleHashEntryData *entry; MemoryContext oldContext; - TupleHashTable saveCurHT; - TupleHashEntryData dummy; bool found; + MinimalTuple key; /* If first time through, clone the input slot to make table slot */ if (hashtable->tableslot == NULL) @@ -356,28 +360,17 @@ LookupTupleHashEntry(TupleHashTable hashtable, TupleTableSlot *slot, /* Need to run the hash functions in short-lived context */ oldContext = MemoryContextSwitchTo(hashtable->tempcxt); - /* - * Set up data needed by hash and match functions - * - * We save and restore CurTupleHashTable just in case someone manages to - * invoke this code re-entrantly. - */ + /* set up data needed by hash and match functions */ hashtable->inputslot = slot; hashtable->in_hash_funcs = hashtable->tab_hash_funcs; hashtable->cur_eq_funcs = hashtable->tab_eq_funcs; - saveCurHT = CurTupleHashTable; - CurTupleHashTable = hashtable; - - /* Search the hash table */ - dummy.firstTuple = NULL; /* flag to reference inputslot */ - entry = (TupleHashEntry) hash_search(hashtable->hashtab, - &dummy, - isnew ? HASH_ENTER : HASH_FIND, - &found); + key = NULL; /* flag to reference inputslot */ if (isnew) { + entry = tuplehash_insert(hashtable->hashtab, key, &found); + if (found) { /* found pre-existing entry */ @@ -385,24 +378,19 @@ LookupTupleHashEntry(TupleHashTable hashtable, TupleTableSlot *slot, } else { - /* - * created new entry - * - * Zero any caller-requested space in the entry. (This zaps the - * "key data" dynahash.c copied into the new entry, but we don't - * care since we're about to overwrite it anyway.) - */ - MemSet(entry, 0, hashtable->entrysize); - - /* Copy the first tuple into the table context */ + /* created new entry */ + *isnew = true; + /* zero caller data */ + entry->additional = NULL; MemoryContextSwitchTo(hashtable->tablecxt); + /* Copy the first tuple into the table context */ entry->firstTuple = ExecCopySlotMinimalTuple(slot); - - *isnew = true; } } - - CurTupleHashTable = saveCurHT; + else + { + entry = tuplehash_lookup(hashtable->hashtab, key); + } MemoryContextSwitchTo(oldContext); @@ -425,34 +413,19 @@ FindTupleHashEntry(TupleHashTable hashtable, TupleTableSlot *slot, { TupleHashEntry entry; MemoryContext oldContext; - TupleHashTable saveCurHT; - TupleHashEntryData dummy; + MinimalTuple key; /* Need to run the hash functions in short-lived context */ oldContext = MemoryContextSwitchTo(hashtable->tempcxt); - /* - * Set up data needed by hash and match functions - * - * We save and restore CurTupleHashTable just in case someone manages to - * invoke this code re-entrantly. - */ + /* Set up data needed by hash and match functions */ hashtable->inputslot = slot; hashtable->in_hash_funcs = hashfunctions; hashtable->cur_eq_funcs = eqfunctions; - saveCurHT = CurTupleHashTable; - CurTupleHashTable = hashtable; - /* Search the hash table */ - dummy.firstTuple = NULL; /* flag to reference inputslot */ - entry = (TupleHashEntry) hash_search(hashtable->hashtab, - &dummy, - HASH_FIND, - NULL); - - CurTupleHashTable = saveCurHT; - + key = NULL; /* flag to reference inputslot */ + entry = tuplehash_lookup(hashtable->hashtab, key); MemoryContextSwitchTo(oldContext); return entry; @@ -468,22 +441,18 @@ FindTupleHashEntry(TupleHashTable hashtable, TupleTableSlot *slot, * This convention avoids the need to materialize virtual input tuples unless * they actually need to get copied into the table. * - * CurTupleHashTable must be set before calling this, since dynahash.c - * doesn't provide any API that would let us get at the hashtable otherwise. - * * Also, the caller must select an appropriate memory context for running * the hash functions. (dynahash.c doesn't change CurrentMemoryContext.) */ static uint32 -TupleHashTableHash(const void *key, Size keysize) +TupleHashTableHash(struct tuplehash_hash *tb, const MinimalTuple tuple) { - MinimalTuple tuple = ((const TupleHashEntryData *) key)->firstTuple; - TupleTableSlot *slot; - TupleHashTable hashtable = CurTupleHashTable; + TupleHashTable hashtable = (TupleHashTable) tb->private; int numCols = hashtable->numCols; AttrNumber *keyColIdx = hashtable->keyColIdx; - FmgrInfo *hashfunctions; uint32 hashkey = 0; + TupleTableSlot *slot; + FmgrInfo *hashfunctions; int i; if (tuple == NULL) @@ -494,8 +463,12 @@ TupleHashTableHash(const void *key, Size keysize) } else { - /* Process a tuple already stored in the table */ - /* (this case never actually occurs in current dynahash.c code) */ + /* + * Process a tuple already stored in the table. + * + * (this case never actually occurs due to the way simplehash.h is + * used, as the hash-value is stored in the entries) + */ slot = hashtable->tableslot; ExecStoreMinimalTuple(tuple, slot, false); hashfunctions = hashtable->tab_hash_funcs; @@ -530,29 +503,21 @@ TupleHashTableHash(const void *key, Size keysize) * * As above, the passed pointers are pointers to TupleHashEntryData. * - * CurTupleHashTable must be set before calling this, since dynahash.c - * doesn't provide any API that would let us get at the hashtable otherwise. - * * Also, the caller must select an appropriate memory context for running * the compare functions. (dynahash.c doesn't change CurrentMemoryContext.) */ static int -TupleHashTableMatch(const void *key1, const void *key2, Size keysize) +TupleHashTableMatch(struct tuplehash_hash *tb, const MinimalTuple tuple1, const MinimalTuple tuple2) { - MinimalTuple tuple1 = ((const TupleHashEntryData *) key1)->firstTuple; - -#ifdef USE_ASSERT_CHECKING - MinimalTuple tuple2 = ((const TupleHashEntryData *) key2)->firstTuple; -#endif TupleTableSlot *slot1; TupleTableSlot *slot2; - TupleHashTable hashtable = CurTupleHashTable; + TupleHashTable hashtable = (TupleHashTable) tb->private; /* - * We assume that dynahash.c will only ever call us with the first + * We assume that simplehash.h will only ever call us with the first * argument being an actual table entry, and the second argument being * LookupTupleHashEntry's dummy TupleHashEntryData. The other direction - * could be supported too, but is not currently used by dynahash.c. + * could be supported too, but is not currently required. */ Assert(tuple1 != NULL); slot1 = hashtable->tableslot; diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c index ce2fc281a4..b06e1c1562 100644 --- a/src/backend/executor/nodeAgg.c +++ b/src/backend/executor/nodeAgg.c @@ -434,20 +434,6 @@ typedef struct AggStatePerPhaseData Sort *sortnode; /* Sort node for input ordering for phase */ } AggStatePerPhaseData; -/* - * To implement hashed aggregation, we need a hashtable that stores a - * representative tuple and an array of AggStatePerGroup structs for each - * distinct set of GROUP BY column values. We compute the hash key from - * the GROUP BY columns. - */ -typedef struct AggHashEntryData *AggHashEntry; - -typedef struct AggHashEntryData -{ - TupleHashEntryData shared; /* common header for hash table entries */ - /* per-aggregate transition status array */ - AggStatePerGroupData pergroup[FLEXIBLE_ARRAY_MEMBER]; -} AggHashEntryData; static void initialize_phase(AggState *aggstate, int newphase); static TupleTableSlot *fetch_input_tuple(AggState *aggstate); @@ -487,7 +473,7 @@ static TupleTableSlot *project_aggregates(AggState *aggstate); static Bitmapset *find_unaggregated_cols(AggState *aggstate); static bool find_unaggregated_cols_walker(Node *node, Bitmapset **colnos); static void build_hash_table(AggState *aggstate); -static AggHashEntry lookup_hash_entry(AggState *aggstate, +static TupleHashEntryData *lookup_hash_entry(AggState *aggstate, TupleTableSlot *inputslot); static TupleTableSlot *agg_retrieve_direct(AggState *aggstate); static void agg_fill_hash_table(AggState *aggstate); @@ -1646,6 +1632,12 @@ find_unaggregated_cols_walker(Node *node, Bitmapset **colnos) /* * Initialize the hash table to empty. * + * To implement hashed aggregation, we need a hashtable that stores a + * representative tuple and an array of AggStatePerGroup structs for each + * distinct set of GROUP BY column values. We compute the hash key from the + * GROUP BY columns. The per-group data is allocated in lookup_hash_entry(), + * for each entry. + * * The hash table always lives in the aggcontext memory context. */ static void @@ -1653,20 +1645,19 @@ build_hash_table(AggState *aggstate) { Agg *node = (Agg *) aggstate->ss.ps.plan; MemoryContext tmpmem = aggstate->tmpcontext->ecxt_per_tuple_memory; - Size entrysize; + Size additionalsize; Assert(node->aggstrategy == AGG_HASHED); Assert(node->numGroups > 0); - entrysize = offsetof(AggHashEntryData, pergroup) + - aggstate->numaggs * sizeof(AggStatePerGroupData); + additionalsize = aggstate->numaggs * sizeof(AggStatePerGroupData); aggstate->hashtable = BuildTupleHashTable(node->numCols, node->grpColIdx, aggstate->phase->eqfunctions, aggstate->hashfunctions, node->numGroups, - entrysize, + additionalsize, aggstate->aggcontexts[0]->ecxt_per_tuple_memory, tmpmem); } @@ -1723,6 +1714,8 @@ find_hash_columns(AggState *aggstate) * * Note that the estimate does not include space for pass-by-reference * transition data values, nor for the representative tuple of each group. + * Nor does this account of the target fill-factor and growth policy of the + * hash table. */ Size hash_agg_entry_size(int numAggs) @@ -1730,11 +1723,10 @@ hash_agg_entry_size(int numAggs) Size entrysize; /* This must match build_hash_table */ - entrysize = offsetof(AggHashEntryData, pergroup) + + entrysize = sizeof(TupleHashEntryData) + numAggs * sizeof(AggStatePerGroupData); entrysize = MAXALIGN(entrysize); - /* Account for hashtable overhead (assuming fill factor = 1) */ - entrysize += 3 * sizeof(void *); + return entrysize; } @@ -1744,12 +1736,12 @@ hash_agg_entry_size(int numAggs) * * When called, CurrentMemoryContext should be the per-query context. */ -static AggHashEntry +static TupleHashEntryData * lookup_hash_entry(AggState *aggstate, TupleTableSlot *inputslot) { TupleTableSlot *hashslot = aggstate->hashslot; ListCell *l; - AggHashEntry entry; + TupleHashEntryData *entry; bool isnew; /* if first time through, initialize hashslot by cloning input slot */ @@ -1771,14 +1763,16 @@ lookup_hash_entry(AggState *aggstate, TupleTableSlot *inputslot) } /* find or create the hashtable entry using the filtered tuple */ - entry = (AggHashEntry) LookupTupleHashEntry(aggstate->hashtable, - hashslot, - &isnew); + entry = LookupTupleHashEntry(aggstate->hashtable, hashslot, &isnew); if (isnew) { + entry->additional = (AggStatePerGroup) + MemoryContextAlloc(aggstate->hashtable->tablecxt, + sizeof(AggStatePerGroupData) * aggstate->numtrans); /* initialize aggregates for new tuple group */ - initialize_aggregates(aggstate, entry->pergroup, 0); + initialize_aggregates(aggstate, (AggStatePerGroup) entry->additional, + 0); } return entry; @@ -2176,7 +2170,7 @@ static void agg_fill_hash_table(AggState *aggstate) { ExprContext *tmpcontext; - AggHashEntry entry; + TupleHashEntryData *entry; TupleTableSlot *outerslot; /* @@ -2203,9 +2197,9 @@ agg_fill_hash_table(AggState *aggstate) /* Advance the aggregates */ if (DO_AGGSPLIT_COMBINE(aggstate->aggsplit)) - combine_aggregates(aggstate, entry->pergroup); + combine_aggregates(aggstate, (AggStatePerGroup) entry->additional); else - advance_aggregates(aggstate, entry->pergroup); + advance_aggregates(aggstate, (AggStatePerGroup) entry->additional); /* Reset per-input-tuple context after each tuple */ ResetExprContext(tmpcontext); @@ -2225,7 +2219,7 @@ agg_retrieve_hash_table(AggState *aggstate) ExprContext *econtext; AggStatePerAgg peragg; AggStatePerGroup pergroup; - AggHashEntry entry; + TupleHashEntryData *entry; TupleTableSlot *firstSlot; TupleTableSlot *result; @@ -2246,7 +2240,7 @@ agg_retrieve_hash_table(AggState *aggstate) /* * Find the next entry in the hash table */ - entry = (AggHashEntry) ScanTupleHashTable(&aggstate->hashiter); + entry = ScanTupleHashTable(aggstate->hashtable, &aggstate->hashiter); if (entry == NULL) { /* No more entries in hashtable, so done */ @@ -2267,11 +2261,11 @@ agg_retrieve_hash_table(AggState *aggstate) * Store the copied first input tuple in the tuple table slot reserved * for it, so that it can be used in ExecProject. */ - ExecStoreMinimalTuple(entry->shared.firstTuple, + ExecStoreMinimalTuple(entry->firstTuple, firstSlot, false); - pergroup = entry->pergroup; + pergroup = (AggStatePerGroup) entry->additional; finalize_aggregates(aggstate, peragg, pergroup, 0); diff --git a/src/backend/executor/nodeRecursiveunion.c b/src/backend/executor/nodeRecursiveunion.c index 39be191c45..acded079e2 100644 --- a/src/backend/executor/nodeRecursiveunion.c +++ b/src/backend/executor/nodeRecursiveunion.c @@ -3,6 +3,10 @@ * nodeRecursiveunion.c * routines to handle RecursiveUnion nodes. * + * To implement UNION (without ALL), we need a hashtable that stores tuples + * already seen. The hash key is computed from the grouping columns. + * + * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * @@ -20,17 +24,6 @@ #include "utils/memutils.h" -/* - * To implement UNION (without ALL), we need a hashtable that stores tuples - * already seen. The hash key is computed from the grouping columns. - */ -typedef struct RUHashEntryData *RUHashEntry; - -typedef struct RUHashEntryData -{ - TupleHashEntryData shared; /* common header for hash table entries */ -} RUHashEntryData; - /* * Initialize the hash table to empty. @@ -48,7 +41,7 @@ build_hash_table(RecursiveUnionState *rustate) rustate->eqfunctions, rustate->hashfunctions, node->numGroups, - sizeof(RUHashEntryData), + 0, rustate->tableContext, rustate->tempContext); } diff --git a/src/backend/executor/nodeSetOp.c b/src/backend/executor/nodeSetOp.c index 633580b436..e94555ead8 100644 --- a/src/backend/executor/nodeSetOp.c +++ b/src/backend/executor/nodeSetOp.c @@ -66,19 +66,6 @@ typedef struct SetOpStatePerGroupData long numRight; /* number of right-input dups in group */ } SetOpStatePerGroupData; -/* - * To implement hashed mode, we need a hashtable that stores a - * representative tuple and the duplicate counts for each distinct set - * of grouping columns. We compute the hash key from the grouping columns. - */ -typedef struct SetOpHashEntryData *SetOpHashEntry; - -typedef struct SetOpHashEntryData -{ - TupleHashEntryData shared; /* common header for hash table entries */ - SetOpStatePerGroupData pergroup; -} SetOpHashEntryData; - static TupleTableSlot *setop_retrieve_direct(SetOpState *setopstate); static void setop_fill_hash_table(SetOpState *setopstate); @@ -141,7 +128,7 @@ build_hash_table(SetOpState *setopstate) setopstate->eqfunctions, setopstate->hashfunctions, node->numGroups, - sizeof(SetOpHashEntryData), + 0, setopstate->tableContext, setopstate->tempContext); } @@ -238,7 +225,7 @@ setop_retrieve_direct(SetOpState *setopstate) * get state info from node */ outerPlan = outerPlanState(setopstate); - pergroup = setopstate->pergroup; + pergroup = (SetOpStatePerGroup) setopstate->pergroup; resultTupleSlot = setopstate->ps.ps_ResultTupleSlot; /* @@ -367,7 +354,7 @@ setop_fill_hash_table(SetOpState *setopstate) { TupleTableSlot *outerslot; int flag; - SetOpHashEntry entry; + TupleHashEntryData *entry; bool isnew; outerslot = ExecProcNode(outerPlan); @@ -383,15 +370,20 @@ setop_fill_hash_table(SetOpState *setopstate) Assert(in_first_rel); /* Find or build hashtable entry for this tuple's group */ - entry = (SetOpHashEntry) - LookupTupleHashEntry(setopstate->hashtable, outerslot, &isnew); + entry = LookupTupleHashEntry(setopstate->hashtable, outerslot, + &isnew); /* If new tuple group, initialize counts */ if (isnew) - initialize_counts(&entry->pergroup); + { + entry->additional = (SetOpStatePerGroup) + MemoryContextAlloc(setopstate->hashtable->tablecxt, + sizeof(SetOpStatePerGroupData)); + initialize_counts((SetOpStatePerGroup) entry->additional); + } /* Advance the counts */ - advance_counts(&entry->pergroup, flag); + advance_counts((SetOpStatePerGroup) entry->additional, flag); } else { @@ -399,12 +391,12 @@ setop_fill_hash_table(SetOpState *setopstate) in_first_rel = false; /* For tuples not seen previously, do not make hashtable entry */ - entry = (SetOpHashEntry) - LookupTupleHashEntry(setopstate->hashtable, outerslot, NULL); + entry = LookupTupleHashEntry(setopstate->hashtable, outerslot, + NULL); /* Advance the counts if entry is already present */ if (entry) - advance_counts(&entry->pergroup, flag); + advance_counts((SetOpStatePerGroup) entry->additional, flag); } /* Must reset temp context after each hashtable lookup */ @@ -422,7 +414,7 @@ setop_fill_hash_table(SetOpState *setopstate) static TupleTableSlot * setop_retrieve_hash_table(SetOpState *setopstate) { - SetOpHashEntry entry; + TupleHashEntryData *entry; TupleTableSlot *resultTupleSlot; /* @@ -438,7 +430,7 @@ setop_retrieve_hash_table(SetOpState *setopstate) /* * Find the next entry in the hash table */ - entry = (SetOpHashEntry) ScanTupleHashTable(&setopstate->hashiter); + entry = ScanTupleHashTable(setopstate->hashtable, &setopstate->hashiter); if (entry == NULL) { /* No more entries in hashtable, so done */ @@ -450,12 +442,12 @@ setop_retrieve_hash_table(SetOpState *setopstate) * See if we should emit any copies of this tuple, and if so return * the first copy. */ - set_output_count(setopstate, &entry->pergroup); + set_output_count(setopstate, (SetOpStatePerGroup) entry->additional); if (setopstate->numOutput > 0) { setopstate->numOutput--; - return ExecStoreMinimalTuple(entry->shared.firstTuple, + return ExecStoreMinimalTuple(entry->firstTuple, resultTupleSlot, false); } diff --git a/src/backend/executor/nodeSubplan.c b/src/backend/executor/nodeSubplan.c index 2cf169f956..8ca8fc460c 100644 --- a/src/backend/executor/nodeSubplan.c +++ b/src/backend/executor/nodeSubplan.c @@ -508,7 +508,7 @@ buildSubPlanHash(SubPlanState *node, ExprContext *econtext) node->tab_eq_funcs, node->tab_hash_funcs, nbuckets, - sizeof(TupleHashEntryData), + 0, node->hashtablecxt, node->hashtempcxt); @@ -527,7 +527,7 @@ buildSubPlanHash(SubPlanState *node, ExprContext *econtext) node->tab_eq_funcs, node->tab_hash_funcs, nbuckets, - sizeof(TupleHashEntryData), + 0, node->hashtablecxt, node->hashtempcxt); } @@ -626,7 +626,7 @@ findPartialMatch(TupleHashTable hashtable, TupleTableSlot *slot, TupleHashEntry entry; InitTupleHashIterator(hashtable, &hashiter); - while ((entry = ScanTupleHashTable(&hashiter)) != NULL) + while ((entry = ScanTupleHashTable(hashtable, &hashiter)) != NULL) { ExecStoreMinimalTuple(entry->firstTuple, hashtable->tableslot, false); if (!execTuplesUnequal(slot, hashtable->tableslot, diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index f657ffc446..644b8b6763 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3292,6 +3292,12 @@ estimate_hashagg_tablesize(Path *path, const AggClauseCosts *agg_costs, /* plus the per-hash-entry overhead */ hashentrysize += hash_agg_entry_size(agg_costs->numAggs); + /* + * Note that this disregards the effect of fill-factor and growth policy + * of the hash-table. That's probably ok, given default the default + * fill-factor is relatively high. It'd be hard to meaningfully factor in + * "double-in-size" growth policies here. + */ return hashentrysize * dNumGroups; } diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index 39521ed08e..136276be53 100644 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -140,7 +140,7 @@ extern void execTuplesHashPrepare(int numCols, extern TupleHashTable BuildTupleHashTable(int numCols, AttrNumber *keyColIdx, FmgrInfo *eqfunctions, FmgrInfo *hashfunctions, - long nbuckets, Size entrysize, + long nbuckets, Size additionalsize, MemoryContext tablecxt, MemoryContext tempcxt); extern TupleHashEntry LookupTupleHashEntry(TupleHashTable hashtable, diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 4fa366178f..f6f73f3c59 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -499,14 +499,23 @@ typedef struct TupleHashTableData *TupleHashTable; typedef struct TupleHashEntryData { - /* firstTuple must be the first field in this struct! */ MinimalTuple firstTuple; /* copy of first tuple in this group */ - /* there may be additional data beyond the end of this struct */ -} TupleHashEntryData; /* VARIABLE LENGTH STRUCT */ + void *additional; /* user data */ + uint32 status; /* hash status */ + uint32 hash; /* hash value (cached) */ +} TupleHashEntryData; + +/* define paramters necessary to generate the tuple hash table interface */ +#define SH_PREFIX tuplehash +#define SH_ELEMENT_TYPE TupleHashEntryData +#define SH_KEY_TYPE MinimalTuple +#define SH_SCOPE extern +#define SH_DECLARE +#include "lib/simplehash.h" typedef struct TupleHashTableData { - HTAB *hashtab; /* underlying dynahash table */ + tuplehash_hash *hashtab; /* underlying hash table */ int numCols; /* number of columns in lookup key */ AttrNumber *keyColIdx; /* attr numbers of key columns */ FmgrInfo *tab_hash_funcs; /* hash functions for table datatype(s) */ @@ -521,7 +530,7 @@ typedef struct TupleHashTableData FmgrInfo *cur_eq_funcs; /* equality functions for input vs. table */ } TupleHashTableData; -typedef HASH_SEQ_STATUS TupleHashIterator; +typedef tuplehash_iterator TupleHashIterator; /* * Use InitTupleHashIterator/TermTupleHashIterator for a read/write scan. @@ -529,16 +538,13 @@ typedef HASH_SEQ_STATUS TupleHashIterator; * explicit scan termination is needed). */ #define InitTupleHashIterator(htable, iter) \ - hash_seq_init(iter, (htable)->hashtab) + tuplehash_start_iterate(htable->hashtab, iter) #define TermTupleHashIterator(iter) \ - hash_seq_term(iter) + ((void) 0) #define ResetTupleHashIterator(htable, iter) \ - do { \ - hash_freeze((htable)->hashtab); \ - hash_seq_init(iter, (htable)->hashtab); \ - } while (0) -#define ScanTupleHashTable(iter) \ - ((TupleHashEntry) hash_seq_search(iter)) + InitTupleHashIterator(htable, iter) +#define ScanTupleHashTable(htable, iter) \ + tuplehash_iterate(htable->hashtab, iter) /* ---------------------------------------------------------------- diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index e61ebbfe2a..6c6d519aac 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2813,6 +2813,8 @@ tsKEY ts_db_fctx ts_tokentype tsearch_readline_state +tuplehash_hash +tuplehash_iterator txid tzEntry u1byte From 9e083fd4683294f41544e6d0d72f6e258ff3a77c Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 17 Oct 2016 11:52:50 +0300 Subject: [PATCH 332/871] Replace PostmasterRandom() with a stronger way of generating randomness. This adds a new routine, pg_strong_random() for generating random bytes, for use in both frontend and backend. At the moment, it's only used in the backend, but the upcoming SCRAM authentication patches need strong random numbers in libpq as well. pg_strong_random() is based on, and replaces, the existing implementation in pgcrypto. It can acquire strong random numbers from a number of sources, depending on what's available: - OpenSSL RAND_bytes(), if built with OpenSSL - On Windows, the native cryptographic functions are used - /dev/urandom - /dev/random Original patch by Magnus Hagander, with further work by Michael Paquier and me. Discussion: --- contrib/pgcrypto/Makefile | 2 +- contrib/pgcrypto/internal.c | 40 +++-- contrib/pgcrypto/random.c | 247 ---------------------------- src/backend/libpq/auth.c | 27 ++- src/backend/postmaster/postmaster.c | 153 +++++------------ src/include/port.h | 3 + src/port/Makefile | 2 +- src/port/pg_strong_random.c | 148 +++++++++++++++++ src/tools/msvc/Mkvcbuild.pm | 6 +- 9 files changed, 244 insertions(+), 384 deletions(-) delete mode 100644 contrib/pgcrypto/random.c create mode 100644 src/port/pg_strong_random.c diff --git a/contrib/pgcrypto/Makefile b/contrib/pgcrypto/Makefile index 805db7626b..76c2f1aa17 100644 --- a/contrib/pgcrypto/Makefile +++ b/contrib/pgcrypto/Makefile @@ -1,7 +1,7 @@ # contrib/pgcrypto/Makefile INT_SRCS = md5.c sha1.c sha2.c internal.c internal-sha2.c blf.c rijndael.c \ - fortuna.c random.c pgp-mpi-internal.c imath.c + fortuna.c pgp-mpi-internal.c imath.c INT_TESTS = sha2 OSSL_SRCS = openssl.c pgp-mpi-openssl.c diff --git a/contrib/pgcrypto/internal.c b/contrib/pgcrypto/internal.c index 02ff976c25..ad942f733a 100644 --- a/contrib/pgcrypto/internal.c +++ b/contrib/pgcrypto/internal.c @@ -626,8 +626,6 @@ static time_t check_time = 0; static void system_reseed(void) { - uint8 buf[1024]; - int n; time_t t; int skip = 1; @@ -642,24 +640,34 @@ system_reseed(void) else if (check_time == 0 || (t - check_time) > SYSTEM_RESEED_CHECK_TIME) { + uint8 buf; + check_time = t; /* roll dice */ - px_get_random_bytes(buf, 1); - skip = buf[0] >= SYSTEM_RESEED_CHANCE; - } - /* clear 1 byte */ - px_memset(buf, 0, sizeof(buf)); - - if (skip) - return; - - n = px_acquire_system_randomness(buf); - if (n > 0) - fortuna_add_entropy(buf, n); + px_get_random_bytes(&buf, 1); + skip = (buf >= SYSTEM_RESEED_CHANCE); - seed_time = t; - px_memset(buf, 0, sizeof(buf)); + /* clear 1 byte */ + px_memset(&buf, 0, sizeof(buf)); + } + if (!skip) + { + /* + * fortuna_add_entropy passes the input to SHA-256, so there's no + * point in giving it more than 256 bits of input to begin with. + */ + uint8 buf[32]; + + if (!pg_strong_random(buf, sizeof(buf))) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not acquire random data"))); + fortuna_add_entropy(buf, sizeof(buf)); + + seed_time = t; + px_memset(buf, 0, sizeof(buf)); + } } int diff --git a/contrib/pgcrypto/random.c b/contrib/pgcrypto/random.c deleted file mode 100644 index d72679e412..0000000000 --- a/contrib/pgcrypto/random.c +++ /dev/null @@ -1,247 +0,0 @@ -/* - * random.c - * Acquire randomness from system. For seeding RNG. - * - * Copyright (c) 2001 Marko Kreen - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * contrib/pgcrypto/random.c - */ - -#include "postgres.h" - -#include "px.h" -#include "utils/memdebug.h" - -/* how many bytes to ask from system random provider */ -#define RND_BYTES 32 - -/* - * Try to read from /dev/urandom or /dev/random on these OS'es. - * - * The list can be pretty liberal, as the device not existing - * is expected event. - */ -#if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) \ - || defined(__NetBSD__) || defined(__DragonFly__) \ - || defined(__darwin__) || defined(__SOLARIS__) \ - || defined(__hpux) || defined(__HPUX__) \ - || defined(__CYGWIN__) || defined(_AIX) - -#define TRY_DEV_RANDOM - -#include -#include - -static int -safe_read(int fd, void *buf, size_t count) -{ - int done = 0; - char *p = buf; - int res; - - while (count) - { - res = read(fd, p, count); - if (res <= 0) - { - if (errno == EINTR) - continue; - return PXE_DEV_READ_ERROR; - } - p += res; - done += res; - count -= res; - } - return done; -} - -static uint8 * -try_dev_random(uint8 *dst) -{ - int fd; - int res; - - fd = open("/dev/urandom", O_RDONLY, 0); - if (fd == -1) - { - fd = open("/dev/random", O_RDONLY, 0); - if (fd == -1) - return dst; - } - res = safe_read(fd, dst, RND_BYTES); - close(fd); - if (res > 0) - dst += res; - return dst; -} -#endif - -/* - * Try to find randomness on Windows - */ -#ifdef WIN32 - -#define TRY_WIN32_GENRAND -#define TRY_WIN32_PERFC - -#include -#include - -/* - * this function is from libtomcrypt - * - * try to use Microsoft crypto API - */ -static uint8 * -try_win32_genrand(uint8 *dst) -{ - int res; - HCRYPTPROV h = 0; - - res = CryptAcquireContext(&h, NULL, MS_DEF_PROV, PROV_RSA_FULL, - (CRYPT_VERIFYCONTEXT | CRYPT_MACHINE_KEYSET)); - if (!res) - res = CryptAcquireContext(&h, NULL, MS_DEF_PROV, PROV_RSA_FULL, - CRYPT_VERIFYCONTEXT | CRYPT_MACHINE_KEYSET | CRYPT_NEWKEYSET); - if (!res) - return dst; - - res = CryptGenRandom(h, RND_BYTES, dst); - if (res == TRUE) - dst += RND_BYTES; - - CryptReleaseContext(h, 0); - return dst; -} - -static uint8 * -try_win32_perfc(uint8 *dst) -{ - int res; - LARGE_INTEGER time; - - res = QueryPerformanceCounter(&time); - if (!res) - return dst; - - memcpy(dst, &time, sizeof(time)); - return dst + sizeof(time); -} -#endif /* WIN32 */ - - -/* - * If we are not on Windows, then hopefully we are - * on a unix-like system. Use the usual suspects - * for randomness. - */ -#ifndef WIN32 - -#define TRY_UNIXSTD - -#include -#include -#include -#include - -/* - * Everything here is predictible, only needs some patience. - * - * But there is a chance that the system-specific functions - * did not work. So keep faith and try to slow the attacker down. - */ -static uint8 * -try_unix_std(uint8 *dst) -{ - pid_t pid; - int x; - PX_MD *md; - struct timeval tv; - int res; - - /* process id */ - pid = getpid(); - memcpy(dst, (uint8 *) &pid, sizeof(pid)); - dst += sizeof(pid); - - /* time */ - gettimeofday(&tv, NULL); - memcpy(dst, (uint8 *) &tv, sizeof(tv)); - dst += sizeof(tv); - - /* pointless, but should not hurt */ - x = random(); - memcpy(dst, (uint8 *) &x, sizeof(x)); - dst += sizeof(x); - - /* hash of uninitialized stack and heap allocations */ - res = px_find_digest("sha1", &md); - if (res >= 0) - { - uint8 *ptr; - uint8 stack[8192]; - int alloc = 32 * 1024; - - VALGRIND_MAKE_MEM_DEFINED(stack, sizeof(stack)); - px_md_update(md, stack, sizeof(stack)); - ptr = px_alloc(alloc); - VALGRIND_MAKE_MEM_DEFINED(ptr, alloc); - px_md_update(md, ptr, alloc); - px_free(ptr); - - px_md_finish(md, dst); - px_md_free(md); - - dst += 20; - } - - return dst; -} -#endif - -/* - * try to extract some randomness for initial seeding - * - * dst should have room for 1024 bytes. - */ -unsigned -px_acquire_system_randomness(uint8 *dst) -{ - uint8 *p = dst; - -#ifdef TRY_DEV_RANDOM - p = try_dev_random(p); -#endif -#ifdef TRY_WIN32_GENRAND - p = try_win32_genrand(p); -#endif -#ifdef TRY_WIN32_PERFC - p = try_win32_perfc(p); -#endif -#ifdef TRY_UNIXSTD - p = try_unix_std(p); -#endif - return p - dst; -} diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index 0ba8530114..44b2212b1d 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -45,6 +45,12 @@ static void auth_failed(Port *port, int status, char *logdetail); static char *recv_password_packet(Port *port); static int recv_and_check_password_packet(Port *port, char **logdetail); +/*---------------------------------------------------------------- + * MD5 authentication + *---------------------------------------------------------------- + */ +static int CheckMD5Auth(Port *port, char **logdetail); + /*---------------------------------------------------------------- * Ident authentication @@ -535,9 +541,7 @@ ClientAuthentication(Port *port) ereport(FATAL, (errcode(ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION), errmsg("MD5 authentication is not supported when \"db_user_namespace\" is enabled"))); - /* include the salt to use for computing the response */ - sendAuthRequest(port, AUTH_REQ_MD5, port->md5Salt, 4); - status = recv_and_check_password_packet(port, &logdetail); + status = CheckMD5Auth(port, &logdetail); break; case uaPassword: @@ -692,10 +696,25 @@ recv_password_packet(Port *port) /*---------------------------------------------------------------- - * MD5 authentication + * MD5 and password authentication *---------------------------------------------------------------- */ +static int +CheckMD5Auth(Port *port, char **logdetail) +{ + /* include the salt to use for computing the response */ + if (!pg_strong_random(port->md5Salt, sizeof(port->md5Salt))) + { + *logdetail = psprintf(_("Could not generate random salt")); + return STATUS_ERROR; + } + + sendAuthRequest(port, AUTH_REQ_MD5, port->md5Salt, 4); + return recv_and_check_password_packet(port, logdetail); +} + + /* * Called when we have sent an authorization request for a password. * Get the response and check it. diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 2d43506cd0..4420138673 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -358,14 +358,6 @@ static volatile bool avlauncher_needs_signal = false; static volatile bool StartWorkerNeeded = true; static volatile bool HaveCrashedWorker = false; -/* - * State for assigning random salts and cancel keys. - * Also, the global MyCancelKey passes the cancel key assigned to a given - * backend from the postmaster to that backend (via fork). - */ -static unsigned int random_seed = 0; -static struct timeval random_start_time; - #ifdef USE_BONJOUR static DNSServiceRef bonjour_sdref = NULL; #endif @@ -403,8 +395,6 @@ static void processCancelRequest(Port *port, void *pkt); static int initMasks(fd_set *rmask); static void report_fork_failure_to_client(Port *port, int errnum); static CAC_state canAcceptConnections(void); -static long PostmasterRandom(void); -static void RandomSalt(char *salt, int len); static void signal_child(pid_t pid, int signal); static bool SignalSomeChildren(int signal, int targets); static void TerminateChildren(int signal); @@ -579,9 +569,11 @@ PostmasterMain(int argc, char *argv[]) * Initialize random(3) so we don't get the same values in every run. * * Note: the seed is pretty predictable from externally-visible facts such - * as postmaster start time, so avoid using random() for security-critical - * random values during postmaster startup. At the time of first - * connection, PostmasterRandom will select a hopefully-more-random seed. + * as postmaster start time, so don't use random() for security-critical + * random values (use pg_strong_random() instead). Backends select a + * somewhat more random seed after forking, in BackendRun(), based on the + * PID and session start timestamp, but that is still not suitable for + * security-critical values. */ srandom((unsigned int) (MyProcPid ^ MyStartTime)); @@ -1292,8 +1284,6 @@ PostmasterMain(int argc, char *argv[]) * Remember postmaster startup time */ PgStartTime = GetCurrentTimestamp(); - /* PostmasterRandom wants its own copy */ - gettimeofday(&random_start_time, NULL); /* * We're ready to rock and roll... @@ -2343,15 +2333,6 @@ ConnCreate(int serverFd) return NULL; } - /* - * Precompute password salt values to use for this connection. It's - * slightly annoying to do this long in advance of knowing whether we'll - * need 'em or not, but we must do the random() calls before we fork, not - * after. Else the postmaster's random sequence won't get advanced, and - * all backends would end up using the same salt... - */ - RandomSalt(port->md5Salt, sizeof(port->md5Salt)); - /* * Allocate GSSAPI specific state struct */ @@ -3904,7 +3885,12 @@ BackendStartup(Port *port) * backend will have its own copy in the forked-off process' value of * MyCancelKey, so that it can transmit the key to the frontend. */ - MyCancelKey = PostmasterRandom(); + if (!pg_strong_random(&MyCancelKey, sizeof(MyCancelKey))) + { + ereport(LOG, + (errmsg("could not generate random query cancel key"))); + return STATUS_ERROR; + } bn->cancel_key = MyCancelKey; /* Pass down canAcceptConnections state */ @@ -4212,13 +4198,6 @@ BackendRun(Port *port) int usecs; int i; - /* - * Don't want backend to be able to see the postmaster random number - * generator state. We have to clobber the static random_seed *and* start - * a new random sequence in the random() library function. - */ - random_seed = 0; - random_start_time.tv_usec = 0; /* slightly hacky way to convert timestamptz into integers */ TimestampDifference(0, port->SessionStartTime, &secs, &usecs); srandom((unsigned int) (MyProcPid ^ (usecs << 12) ^ secs)); @@ -5066,66 +5045,6 @@ StartupPacketTimeoutHandler(void) } -/* - * RandomSalt - */ -static void -RandomSalt(char *salt, int len) -{ - long rand; - int i; - - /* - * We use % 255, sacrificing one possible byte value, so as to ensure that - * all bits of the random() value participate in the result. While at it, - * add one to avoid generating any null bytes. - */ - for (i = 0; i < len; i++) - { - rand = PostmasterRandom(); - salt[i] = (rand % 255) + 1; - } -} - -/* - * PostmasterRandom - * - * Caution: use this only for values needed during connection-request - * processing. Otherwise, the intended property of having an unpredictable - * delay between random_start_time and random_stop_time will be broken. - */ -static long -PostmasterRandom(void) -{ - /* - * Select a random seed at the time of first receiving a request. - */ - if (random_seed == 0) - { - do - { - struct timeval random_stop_time; - - gettimeofday(&random_stop_time, NULL); - - /* - * We are not sure how much precision is in tv_usec, so we swap - * the high and low 16 bits of 'random_stop_time' and XOR them - * with 'random_start_time'. On the off chance that the result is - * 0, we loop until it isn't. - */ - random_seed = random_start_time.tv_usec ^ - ((random_stop_time.tv_usec << 16) | - ((random_stop_time.tv_usec >> 16) & 0xffff)); - } - while (random_seed == 0); - - srandom(random_seed); - } - - return random(); -} - /* * Count up number of child processes of specified types (dead_end chidren * are always excluded). @@ -5303,31 +5222,37 @@ StartAutovacuumWorker(void) * we'd better have something random in the field to prevent * unfriendly people from sending cancels to them. */ - MyCancelKey = PostmasterRandom(); - bn->cancel_key = MyCancelKey; + if (pg_strong_random(&MyCancelKey, sizeof(MyCancelKey))) + { + bn->cancel_key = MyCancelKey; - /* Autovac workers are not dead_end and need a child slot */ - bn->dead_end = false; - bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); - bn->bgworker_notify = false; + /* Autovac workers are not dead_end and need a child slot */ + bn->dead_end = false; + bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); + bn->bgworker_notify = false; - bn->pid = StartAutoVacWorker(); - if (bn->pid > 0) - { - bn->bkend_type = BACKEND_TYPE_AUTOVAC; - dlist_push_head(&BackendList, &bn->elem); + bn->pid = StartAutoVacWorker(); + if (bn->pid > 0) + { + bn->bkend_type = BACKEND_TYPE_AUTOVAC; + dlist_push_head(&BackendList, &bn->elem); #ifdef EXEC_BACKEND - ShmemBackendArrayAdd(bn); + ShmemBackendArrayAdd(bn); #endif - /* all OK */ - return; + /* all OK */ + return; + } + + /* + * fork failed, fall through to report -- actual error message + * was logged by StartAutoVacWorker + */ + (void) ReleasePostmasterChildSlot(bn->child_slot); } + else + ereport(LOG, + (errmsg("could not generate random query cancel key"))); - /* - * fork failed, fall through to report -- actual error message was - * logged by StartAutoVacWorker - */ - (void) ReleasePostmasterChildSlot(bn->child_slot); free(bn); } else @@ -5615,7 +5540,11 @@ assign_backendlist_entry(RegisteredBgWorker *rw) * have something random in the field to prevent unfriendly people from * sending cancels to them. */ - MyCancelKey = PostmasterRandom(); + if (!pg_strong_random(&MyCancelKey, sizeof(MyCancelKey))) + { + rw->rw_crashed_at = GetCurrentTimestamp(); + return false; + } bn->cancel_key = MyCancelKey; bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); diff --git a/src/include/port.h b/src/include/port.h index b81fa4a89e..4bb9feeb01 100644 --- a/src/include/port.h +++ b/src/include/port.h @@ -454,6 +454,9 @@ extern int pg_codepage_to_encoding(UINT cp); extern char *inet_net_ntop(int af, const void *src, int bits, char *dst, size_t size); +/* port/pg_strong_random.c */ +extern bool pg_strong_random(void *buf, size_t len); + /* port/pgcheckdir.c */ extern int pg_check_dir(const char *dir); diff --git a/src/port/Makefile b/src/port/Makefile index bc9b63add0..d34f409ee9 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -32,7 +32,7 @@ LIBS += $(PTHREAD_LIBS) OBJS = $(LIBOBJS) $(PG_CRC32C_OBJS) chklocale.o erand48.o inet_net_ntop.o \ noblock.o path.o pgcheckdir.o pgmkdirp.o pgsleep.o \ - pgstrcasecmp.o pqsignal.o \ + pg_strong_random.o pgstrcasecmp.o pqsignal.o \ qsort.o qsort_arg.o quotes.o sprompt.o tar.o thread.o # foo_srv.o and foo.o are both built from foo.c, but only foo.o has -DFRONTEND diff --git a/src/port/pg_strong_random.c b/src/port/pg_strong_random.c new file mode 100644 index 0000000000..a404111d74 --- /dev/null +++ b/src/port/pg_strong_random.c @@ -0,0 +1,148 @@ +/*------------------------------------------------------------------------- + * + * pg_strong_random.c + * pg_strong_random() function to return a strong random number + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/port/pg_strong_random.c + * + *------------------------------------------------------------------------- + */ + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include +#include + +#ifdef USE_SSL +#include +#endif +#ifdef WIN32 +#include +#endif + +static bool random_from_file(char *filename, void *buf, size_t len); + +#ifdef WIN32 +/* + * Cache a global crypto provider that only gets freed when the process + * exits, in case we need random numbers more than once. + */ +static HCRYPTPROV hProvider = 0; +#endif + +/* + * Read (random) bytes from a file. + */ +static bool +random_from_file(char *filename, void *buf, size_t len) +{ + int f; + char *p = buf; + ssize_t res; + + f = open(filename, O_RDONLY, 0); + if (f == -1) + return false; + + while (len) + { + res = read(f, p, len); + if (res <= 0) + { + if (errno == EINTR) + continue; /* interrupted by signal, just retry */ + + close(f); + return false; + } + + p += res; + len -= res; + } + + close(f); + return true; +} + +/* + * pg_strong_random + * + * Generate requested number of random bytes. The bytes are + * cryptographically strong random, suitable for use e.g. in key + * generation. + * + * The bytes can be acquired from a number of sources, depending + * on what's available. We try the following, in this order: + * + * 1. OpenSSL's RAND_bytes() + * 2. Windows' CryptGenRandom() function + * 3. /dev/urandom + * 4. /dev/random + * + * Returns true on success, and false if none of the sources + * were available. NB: It is important to check the return value! + * Proceeding with key generation when no random data was available + * would lead to predictable keys and security issues. + */ +bool +pg_strong_random(void *buf, size_t len) +{ +#ifdef USE_SSL + + /* + * When built with OpenSSL, first try the random generation function from + * there. + */ + if (RAND_bytes(buf, len) == 1) + return true; +#endif + +#ifdef WIN32 + + /* + * Windows has CryptoAPI for strong cryptographic numbers. + */ + if (hProvider == 0) + { + if (!CryptAcquireContext(&hProvider, + NULL, + MS_DEF_PROV, + PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) + { + /* + * On failure, set back to 0 in case the value was for some reason + * modified. + */ + hProvider = 0; + } + } + + /* Re-check in case we just retrieved the provider */ + if (hProvider != 0) + { + if (CryptGenRandom(hProvider, len, buf)) + return true; + } +#endif + + /* + * If there is no OpenSSL and no CryptoAPI (or they didn't work), then + * fall back on reading /dev/urandom or even /dev/random. + */ + if (random_from_file("/dev/urandom", buf, len)) + return true; + if (random_from_file("/dev/random", buf, len)) + return true; + + /* None of the sources were available. */ + return false; +} diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index de764dd4d4..e6c4aef99f 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -92,7 +92,7 @@ sub mkvcbuild srandom.c getaddrinfo.c gettimeofday.c inet_net_ntop.c kill.c open.c erand48.c snprintf.c strlcat.c strlcpy.c dirmod.c noblock.c path.c pgcheckdir.c pgmkdirp.c pgsleep.c pgstrcasecmp.c pqsignal.c - mkdtemp.c qsort.c qsort_arg.c quotes.c system.c + mkdtemp.c pg_strong_random.c qsort.c qsort_arg.c quotes.c system.c sprompt.c tar.c thread.c getopt.c getopt_long.c dirent.c win32env.c win32error.c win32security.c win32setlocale.c); @@ -425,8 +425,8 @@ sub mkvcbuild 'sha1.c', 'sha2.c', 'internal.c', 'internal-sha2.c', 'blf.c', 'rijndael.c', - 'fortuna.c', 'random.c', - 'pgp-mpi-internal.c', 'imath.c'); + 'fortuna.c', 'pgp-mpi-internal.c', + 'imath.c'); } $pgcrypto->AddReference($postgres); $pgcrypto->AddLibrary('ws2_32.lib'); From d8589946ddd5c43e1ebd01c5e92d0e177cbfc198 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 17 Oct 2016 12:13:16 +0300 Subject: [PATCH 333/871] Fix use-after-free around DISTINCT transition function calls. Have tuplesort_gettupleslot() copy the contents of its current table slot as needed. This is based on an approach taken by tuplestore_gettupleslot(). In the future, tuplesort_gettupleslot() may also be taught to avoid copying the tuple where caller can determine that that is safe (the tuplestore_gettupleslot() interface already offers this option to callers). Patch by Peter Geoghegan. Fixes bug #14344, reported by Regina Obe. Report: <20160929035538.20224.39628@wrigleys.postgresql.org> Backpatch-through: 9.6 --- src/backend/utils/sort/tuplesort.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index f6eb30c2ce..46512cfca1 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -2084,6 +2084,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward, * determination of "non-equal tuple" based on simple binary inequality. A * NULL value in leading attribute will set abbreviated value to zeroed * representation, which caller may rely on in abbreviated inequality check. + * + * The slot receives a copied tuple (sometimes allocated in caller memory + * context) that will stay valid regardless of future manipulations of the + * tuplesort's state. */ bool tuplesort_gettupleslot(Tuplesortstate *state, bool forward, @@ -2104,6 +2108,11 @@ tuplesort_gettupleslot(Tuplesortstate *state, bool forward, if (state->sortKeys->abbrev_converter && abbrev) *abbrev = stup.datum1; + if (!should_free) + { + stup.tuple = heap_copy_minimal_tuple((MinimalTuple) stup.tuple); + should_free = true; + } ExecStoreMinimalTuple((MinimalTuple) stup.tuple, slot, should_free); return true; } From 5ff4a67f63fd6d3eb01ff9707d4674ed54a89f3b Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 17 Oct 2016 17:29:33 +0300 Subject: [PATCH 334/871] Use OpenSSL EVP API for symmetric encryption in pgcrypto. The old "low-level" API is deprecated, and doesn't support hardware acceleration. And this makes the code simpler, too. Discussion: <561274F1.1030000@iki.fi> --- contrib/pgcrypto/openssl.c | 554 ++++++++++++------------------------- 1 file changed, 178 insertions(+), 376 deletions(-) diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index 851cb616cb..f3e3a92486 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -34,12 +34,8 @@ #include "px.h" #include -#include -#include -#include -#include -#include #include +#include #include "utils/memutils.h" #include "utils/resowner.h" @@ -235,47 +231,27 @@ px_find_digest(const char *name, PX_MD **res) /* * Ciphers * - * The problem with OpenSSL is that the EVP* family - * of functions does not allow enough flexibility - * and forces some of the parameters (keylen, - * padding) to SSL defaults. - * - * So need to manage ciphers ourselves. + * We use OpenSSL's EVP* family of functions for these. + */ + +/* + * prototype for the EVP functions that return an algorithm, e.g. + * EVP_aes_128_cbc(). */ +typedef const EVP_CIPHER *(*ossl_EVP_cipher_func)(void); struct ossl_cipher { int (*init) (PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv); - int (*encrypt) (PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res); - int (*decrypt) (PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res); - + ossl_EVP_cipher_func cipher_func; int block_size; int max_key_size; - int stream_cipher; }; typedef struct { - union - { - struct - { - BF_KEY key; - int num; - } bf; - struct - { - DES_key_schedule key_schedule; - } des; - struct - { - DES_key_schedule k1, - k2, - k3; - } des3; - CAST_KEY cast_key; - AES_KEY aes_key; - } u; + EVP_CIPHER_CTX evp_ctx; + const EVP_CIPHER *evp_ciph; uint8 key[MAX_KEY]; uint8 iv[MAX_IV]; unsigned klen; @@ -283,7 +259,7 @@ typedef struct const struct ossl_cipher *ciph; } ossldata; -/* generic */ +/* Common routines for all algorithms */ static unsigned gen_ossl_block_size(PX_Cipher *c) @@ -316,11 +292,62 @@ gen_ossl_free(PX_Cipher *c) { ossldata *od = (ossldata *) c->ptr; + EVP_CIPHER_CTX_cleanup(&od->evp_ctx); px_memset(od, 0, sizeof(*od)); px_free(od); px_free(c); } +static int +gen_ossl_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, + uint8 *res) +{ + ossldata *od = c->ptr; + int outlen; + + if (!od->init) + { + EVP_CIPHER_CTX_init(&od->evp_ctx); + if (!EVP_DecryptInit_ex(&od->evp_ctx, od->evp_ciph, NULL, NULL, NULL)) + return PXE_CIPHER_INIT; + if (!EVP_CIPHER_CTX_set_key_length(&od->evp_ctx, od->klen)) + return PXE_CIPHER_INIT; + if (!EVP_DecryptInit_ex(&od->evp_ctx, NULL, NULL, od->key, od->iv)) + return PXE_CIPHER_INIT; + od->init = true; + } + + if (!EVP_DecryptUpdate(&od->evp_ctx, res, &outlen, data, dlen)) + return PXE_DECRYPT_FAILED; + + return 0; +} + +static int +gen_ossl_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, + uint8 *res) +{ + ossldata *od = c->ptr; + int outlen; + + if (!od->init) + { + EVP_CIPHER_CTX_init(&od->evp_ctx); + if (!EVP_EncryptInit_ex(&od->evp_ctx, od->evp_ciph, NULL, NULL, NULL)) + return PXE_CIPHER_INIT; + if (!EVP_CIPHER_CTX_set_key_length(&od->evp_ctx, od->klen)) + return PXE_CIPHER_INIT; + if (!EVP_EncryptInit_ex(&od->evp_ctx, NULL, NULL, od->key, od->iv)) + return PXE_CIPHER_INIT; + od->init = true; + } + + if (!EVP_EncryptUpdate(&od->evp_ctx, res, &outlen, data, dlen)) + return PXE_ERR_GENERIC; + + return 0; +} + /* Blowfish */ /* @@ -342,13 +369,21 @@ bf_check_supported_key_len(void) static const uint8 data[8] = {0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}; static const uint8 res[8] = {0xc0, 0x45, 0x04, 0x01, 0x2e, 0x4e, 0x1f, 0x53}; - static uint8 out[8]; - - BF_KEY bf_key; + uint8 out[8]; + EVP_CIPHER_CTX evp_ctx; + int outlen; /* encrypt with 448bits key and verify output */ - BF_set_key(&bf_key, 56, key); - BF_ecb_encrypt(data, out, &bf_key, BF_ENCRYPT); + EVP_CIPHER_CTX_init(&evp_ctx); + if (!EVP_EncryptInit_ex(&evp_ctx, EVP_bf_ecb(), NULL, NULL, NULL)) + return 0; + if (!EVP_CIPHER_CTX_set_key_length(&evp_ctx, 56)) + return 0; + if (!EVP_EncryptInit_ex(&evp_ctx, NULL, NULL, key, NULL)) + return 0; + + if (!EVP_EncryptUpdate(&evp_ctx, out, &outlen, data, 8)) + return 0; if (memcmp(out, res, 8) != 0) return 0; /* Output does not match -> strong cipher is @@ -360,6 +395,7 @@ static int bf_init(PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv) { ossldata *od = c->ptr; + unsigned bs = gen_ossl_block_size(c); static int bf_is_strong = -1; /* @@ -375,74 +411,13 @@ bf_init(PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv) return PXE_KEY_TOO_BIG; /* Key len is supported. We can use it. */ - BF_set_key(&od->u.bf.key, klen, key); + od->klen = klen; + memcpy(od->key, key, klen); + if (iv) - memcpy(od->iv, iv, BF_BLOCK); + memcpy(od->iv, iv, bs); else - memset(od->iv, 0, BF_BLOCK); - od->u.bf.num = 0; - return 0; -} - -static int -bf_ecb_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - unsigned bs = gen_ossl_block_size(c); - unsigned i; - ossldata *od = c->ptr; - - for (i = 0; i < dlen / bs; i++) - BF_ecb_encrypt(data + i * bs, res + i * bs, &od->u.bf.key, BF_ENCRYPT); - return 0; -} - -static int -bf_ecb_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - unsigned bs = gen_ossl_block_size(c), - i; - ossldata *od = c->ptr; - - for (i = 0; i < dlen / bs; i++) - BF_ecb_encrypt(data + i * bs, res + i * bs, &od->u.bf.key, BF_DECRYPT); - return 0; -} - -static int -bf_cbc_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - ossldata *od = c->ptr; - - BF_cbc_encrypt(data, res, dlen, &od->u.bf.key, od->iv, BF_ENCRYPT); - return 0; -} - -static int -bf_cbc_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - ossldata *od = c->ptr; - - BF_cbc_encrypt(data, res, dlen, &od->u.bf.key, od->iv, BF_DECRYPT); - return 0; -} - -static int -bf_cfb64_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - ossldata *od = c->ptr; - - BF_cfb64_encrypt(data, res, dlen, &od->u.bf.key, od->iv, - &od->u.bf.num, BF_ENCRYPT); - return 0; -} - -static int -bf_cfb64_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - ossldata *od = c->ptr; - - BF_cfb64_encrypt(data, res, dlen, &od->u.bf.key, od->iv, - &od->u.bf.num, BF_DECRYPT); + memset(od->iv, 0, bs); return 0; } @@ -452,69 +427,16 @@ static int ossl_des_init(PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv) { ossldata *od = c->ptr; - DES_cblock xkey; + unsigned bs = gen_ossl_block_size(c); - memset(&xkey, 0, sizeof(xkey)); - memcpy(&xkey, key, klen > 8 ? 8 : klen); - DES_set_key(&xkey, &od->u.des.key_schedule); - memset(&xkey, 0, sizeof(xkey)); + od->klen = 8; + memset(od->key, 0, 8); + memcpy(od->key, key, klen > 8 ? 8 : klen); if (iv) - memcpy(od->iv, iv, 8); + memcpy(od->iv, iv, bs); else - memset(od->iv, 0, 8); - return 0; -} - -static int -ossl_des_ecb_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - unsigned bs = gen_ossl_block_size(c); - unsigned i; - ossldata *od = c->ptr; - - for (i = 0; i < dlen / bs; i++) - DES_ecb_encrypt((DES_cblock *) (data + i * bs), - (DES_cblock *) (res + i * bs), - &od->u.des.key_schedule, 1); - return 0; -} - -static int -ossl_des_ecb_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - unsigned bs = gen_ossl_block_size(c); - unsigned i; - ossldata *od = c->ptr; - - for (i = 0; i < dlen / bs; i++) - DES_ecb_encrypt((DES_cblock *) (data + i * bs), - (DES_cblock *) (res + i * bs), - &od->u.des.key_schedule, 0); - return 0; -} - -static int -ossl_des_cbc_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - ossldata *od = c->ptr; - - DES_ncbc_encrypt(data, res, dlen, &od->u.des.key_schedule, - (DES_cblock *) od->iv, 1); - return 0; -} - -static int -ossl_des_cbc_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - ossldata *od = c->ptr; - - DES_ncbc_encrypt(data, res, dlen, &od->u.des.key_schedule, - (DES_cblock *) od->iv, 0); + memset(od->iv, 0, bs); return 0; } @@ -524,82 +446,16 @@ static int ossl_des3_init(PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv) { ossldata *od = c->ptr; - DES_cblock xkey1, - xkey2, - xkey3; - - memset(&xkey1, 0, sizeof(xkey1)); - memset(&xkey2, 0, sizeof(xkey2)); - memset(&xkey3, 0, sizeof(xkey3)); - memcpy(&xkey1, key, klen > 8 ? 8 : klen); - if (klen > 8) - memcpy(&xkey2, key + 8, (klen - 8) > 8 ? 8 : (klen - 8)); - if (klen > 16) - memcpy(&xkey3, key + 16, (klen - 16) > 8 ? 8 : (klen - 16)); - - DES_set_key(&xkey1, &od->u.des3.k1); - DES_set_key(&xkey2, &od->u.des3.k2); - DES_set_key(&xkey3, &od->u.des3.k3); - memset(&xkey1, 0, sizeof(xkey1)); - memset(&xkey2, 0, sizeof(xkey2)); - memset(&xkey3, 0, sizeof(xkey3)); - - if (iv) - memcpy(od->iv, iv, 8); - else - memset(od->iv, 0, 8); - return 0; -} - -static int -ossl_des3_ecb_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - unsigned bs = gen_ossl_block_size(c); - unsigned i; - ossldata *od = c->ptr; - - for (i = 0; i < dlen / bs; i++) - DES_ecb3_encrypt((void *) (data + i * bs), (void *) (res + i * bs), - &od->u.des3.k1, &od->u.des3.k2, &od->u.des3.k3, 1); - return 0; -} - -static int -ossl_des3_ecb_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ unsigned bs = gen_ossl_block_size(c); - unsigned i; - ossldata *od = c->ptr; - for (i = 0; i < dlen / bs; i++) - DES_ecb3_encrypt((void *) (data + i * bs), (void *) (res + i * bs), - &od->u.des3.k1, &od->u.des3.k2, &od->u.des3.k3, 0); - return 0; -} + od->klen = 24; + memset(od->key, 0, 24); + memcpy(od->key, key, klen > 24 ? 24 : klen); -static int -ossl_des3_cbc_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - ossldata *od = c->ptr; - - DES_ede3_cbc_encrypt(data, res, dlen, - &od->u.des3.k1, &od->u.des3.k2, &od->u.des3.k3, - (DES_cblock *) od->iv, 1); - return 0; -} - -static int -ossl_des3_cbc_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - ossldata *od = c->ptr; - - DES_ede3_cbc_encrypt(data, res, dlen, - &od->u.des3.k1, &od->u.des3.k2, &od->u.des3.k3, - (DES_cblock *) od->iv, 0); + if (iv) + memcpy(od->iv, iv, bs); + else + memset(od->iv, 0, bs); return 0; } @@ -611,7 +467,9 @@ ossl_cast_init(PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv) ossldata *od = c->ptr; unsigned bs = gen_ossl_block_size(c); - CAST_set_key(&od->u.cast_key, klen, key); + od->klen = klen; + memcpy(od->key, key, klen); + if (iv) memcpy(od->iv, iv, bs); else @@ -619,48 +477,6 @@ ossl_cast_init(PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv) return 0; } -static int -ossl_cast_ecb_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - unsigned bs = gen_ossl_block_size(c); - ossldata *od = c->ptr; - const uint8 *end = data + dlen - bs; - - for (; data <= end; data += bs, res += bs) - CAST_ecb_encrypt(data, res, &od->u.cast_key, CAST_ENCRYPT); - return 0; -} - -static int -ossl_cast_ecb_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - unsigned bs = gen_ossl_block_size(c); - ossldata *od = c->ptr; - const uint8 *end = data + dlen - bs; - - for (; data <= end; data += bs, res += bs) - CAST_ecb_encrypt(data, res, &od->u.cast_key, CAST_DECRYPT); - return 0; -} - -static int -ossl_cast_cbc_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - ossldata *od = c->ptr; - - CAST_cbc_encrypt(data, res, dlen, &od->u.cast_key, od->iv, CAST_ENCRYPT); - return 0; -} - -static int -ossl_cast_cbc_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, uint8 *res) -{ - ossldata *od = c->ptr; - - CAST_cbc_encrypt(data, res, dlen, &od->u.cast_key, od->iv, CAST_DECRYPT); - return 0; -} - /* AES */ static int @@ -684,96 +500,68 @@ ossl_aes_init(PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv) memcpy(od->iv, iv, bs); else memset(od->iv, 0, bs); + return 0; } static int -ossl_aes_key_init(ossldata *od, int type) +ossl_aes_ecb_init(PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv) { + ossldata *od = c->ptr; int err; - /* - * Strong key support could be missing on some openssl installations. We - * must check return value from set key function. - */ - if (type == AES_ENCRYPT) - err = AES_set_encrypt_key(od->key, od->klen * 8, &od->u.aes_key); - else - err = AES_set_decrypt_key(od->key, od->klen * 8, &od->u.aes_key); + err = ossl_aes_init(c, key, klen, iv); + if (err) + return err; - if (err == 0) + switch (od->klen) { - od->init = 1; - return 0; + case 128 / 8: + od->evp_ciph = EVP_aes_128_ecb(); + break; + case 192 / 8: + od->evp_ciph = EVP_aes_192_ecb(); + break; + case 256 / 8: + od->evp_ciph = EVP_aes_256_ecb(); + break; + default: + /* shouldn't happen */ + err = PXE_CIPHER_INIT; + break; } - od->init = 0; - return PXE_KEY_TOO_BIG; -} -static int -ossl_aes_ecb_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - unsigned bs = gen_ossl_block_size(c); - ossldata *od = c->ptr; - const uint8 *end = data + dlen - bs; - int err; - - if (!od->init) - if ((err = ossl_aes_key_init(od, AES_ENCRYPT)) != 0) - return err; - - for (; data <= end; data += bs, res += bs) - AES_ecb_encrypt(data, res, &od->u.aes_key, AES_ENCRYPT); - return 0; + return err; } static int -ossl_aes_ecb_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) +ossl_aes_cbc_init(PX_Cipher *c, const uint8 *key, unsigned klen, const uint8 *iv) { - unsigned bs = gen_ossl_block_size(c); ossldata *od = c->ptr; - const uint8 *end = data + dlen - bs; int err; - if (!od->init) - if ((err = ossl_aes_key_init(od, AES_DECRYPT)) != 0) - return err; - - for (; data <= end; data += bs, res += bs) - AES_ecb_encrypt(data, res, &od->u.aes_key, AES_DECRYPT); - return 0; -} - -static int -ossl_aes_cbc_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - ossldata *od = c->ptr; - int err; - - if (!od->init) - if ((err = ossl_aes_key_init(od, AES_ENCRYPT)) != 0) - return err; - - AES_cbc_encrypt(data, res, dlen, &od->u.aes_key, od->iv, AES_ENCRYPT); - return 0; -} + err = ossl_aes_init(c, key, klen, iv); + if (err) + return err; -static int -ossl_aes_cbc_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, - uint8 *res) -{ - ossldata *od = c->ptr; - int err; - - if (!od->init) - if ((err = ossl_aes_key_init(od, AES_DECRYPT)) != 0) - return err; + switch (od->klen) + { + case 128 / 8: + od->evp_ciph = EVP_aes_128_cbc(); + break; + case 192 / 8: + od->evp_ciph = EVP_aes_192_cbc(); + break; + case 256 / 8: + od->evp_ciph = EVP_aes_256_cbc(); + break; + default: + /* shouldn't happen */ + err = PXE_CIPHER_INIT; + break; + } - AES_cbc_encrypt(data, res, dlen, &od->u.aes_key, od->iv, AES_DECRYPT); - return 0; + return err; } /* @@ -799,58 +587,69 @@ static PX_Alias ossl_aliases[] = { }; static const struct ossl_cipher ossl_bf_cbc = { - bf_init, bf_cbc_encrypt, bf_cbc_decrypt, - 64 / 8, 448 / 8, 0 + bf_init, + EVP_bf_cbc, + 64 / 8, 448 / 8 }; static const struct ossl_cipher ossl_bf_ecb = { - bf_init, bf_ecb_encrypt, bf_ecb_decrypt, - 64 / 8, 448 / 8, 0 + bf_init, + EVP_bf_ecb, + 64 / 8, 448 / 8 }; static const struct ossl_cipher ossl_bf_cfb = { - bf_init, bf_cfb64_encrypt, bf_cfb64_decrypt, - 64 / 8, 448 / 8, 1 + bf_init, + EVP_bf_cfb, + 64 / 8, 448 / 8 }; static const struct ossl_cipher ossl_des_ecb = { - ossl_des_init, ossl_des_ecb_encrypt, ossl_des_ecb_decrypt, - 64 / 8, 64 / 8, 0 + ossl_des_init, + EVP_des_ecb, + 64 / 8, 64 / 8 }; static const struct ossl_cipher ossl_des_cbc = { - ossl_des_init, ossl_des_cbc_encrypt, ossl_des_cbc_decrypt, - 64 / 8, 64 / 8, 0 + ossl_des_init, + EVP_des_cbc, + 64 / 8, 64 / 8 }; static const struct ossl_cipher ossl_des3_ecb = { - ossl_des3_init, ossl_des3_ecb_encrypt, ossl_des3_ecb_decrypt, - 64 / 8, 192 / 8, 0 + ossl_des3_init, + EVP_des_ede3_ecb, + 64 / 8, 192 / 8 }; static const struct ossl_cipher ossl_des3_cbc = { - ossl_des3_init, ossl_des3_cbc_encrypt, ossl_des3_cbc_decrypt, - 64 / 8, 192 / 8, 0 + ossl_des3_init, + EVP_des_ede3_cbc, + 64 / 8, 192 / 8 }; static const struct ossl_cipher ossl_cast_ecb = { - ossl_cast_init, ossl_cast_ecb_encrypt, ossl_cast_ecb_decrypt, - 64 / 8, 128 / 8, 0 + ossl_cast_init, + EVP_cast5_ecb, + 64 / 8, 128 / 8 }; static const struct ossl_cipher ossl_cast_cbc = { - ossl_cast_init, ossl_cast_cbc_encrypt, ossl_cast_cbc_decrypt, - 64 / 8, 128 / 8, 0 + ossl_cast_init, + EVP_cast5_cbc, + 64 / 8, 128 / 8 }; static const struct ossl_cipher ossl_aes_ecb = { - ossl_aes_init, ossl_aes_ecb_encrypt, ossl_aes_ecb_decrypt, - 128 / 8, 256 / 8, 0 + ossl_aes_ecb_init, + NULL, /* EVP_aes_XXX_ecb(), determined in init function */ + 128 / 8, 256 / 8 }; static const struct ossl_cipher ossl_aes_cbc = { - ossl_aes_init, ossl_aes_cbc_encrypt, ossl_aes_cbc_decrypt, - 128 / 8, 256 / 8, 0 + ossl_aes_cbc_init, + NULL, /* EVP_aes_XXX_cbc(), determined in init function */ + 128 / 8, 256 / 8 }; /* @@ -897,14 +696,17 @@ px_find_cipher(const char *name, PX_Cipher **res) memset(od, 0, sizeof(*od)); od->ciph = i->ciph; + if (i->ciph->cipher_func) + od->evp_ciph = i->ciph->cipher_func(); + c = px_alloc(sizeof(*c)); c->block_size = gen_ossl_block_size; c->key_size = gen_ossl_key_size; c->iv_size = gen_ossl_iv_size; c->free = gen_ossl_free; c->init = od->ciph->init; - c->encrypt = od->ciph->encrypt; - c->decrypt = od->ciph->decrypt; + c->encrypt = gen_ossl_encrypt; + c->decrypt = gen_ossl_decrypt; c->ptr = od; *res = c; From 7d3235ba42f8d5fc70c58e242702cc5e2e3549a6 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 17 Oct 2016 16:31:13 -0400 Subject: [PATCH 335/871] By default, set log_line_prefix = '%m [%p] '. This value might not be to everyone's taste; in particular, some people might prefer %t to %m, and others may want %u, %d, or other fields. However, it's a vast improvement on the old default of ''. Christoph Berg --- doc/src/sgml/config.sgml | 14 +++++++++++++- src/backend/utils/misc/guc.c | 2 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e826c19698..99ff9f5ab5 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -5004,7 +5004,8 @@ local0.* /var/log/postgresql value will pad on the left. Padding can be useful to aid human readability in log files. This parameter can only be set in the postgresql.conf - file or on the server command line. The default is an empty string. + file or on the server command line. The default is + '%m [%p] ' which logs a time stamp and the process ID. @@ -5142,6 +5143,17 @@ FROM pg_stat_activity; include those escapes if you are logging to syslog. + + + + The %q escape is useful when including information that is + only available in session (backend) context like user or database + name. For example: + +log_line_prefix = '%m [%p] %q%u@%d/%a ' + + + diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 622279b058..65660c1bf7 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3014,7 +3014,7 @@ static struct config_string ConfigureNamesString[] = gettext_noop("If blank, no prefix is used.") }, &Log_line_prefix, - "", + "%m [%p] ", NULL, NULL, NULL }, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 05b1373594..159ada3bc6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -430,7 +430,7 @@ #log_duration = off #log_error_verbosity = default # terse, default, or verbose messages #log_hostname = off -#log_line_prefix = '' # special values: +#log_line_prefix = '%m [%p] ' # special values: # %a = application name # %u = user name # %d = database name From faae1c918e8aaae034eaf3ea103fcb6ba9adc5ab Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 18 Oct 2016 16:28:23 +0300 Subject: [PATCH 336/871] Revert "Replace PostmasterRandom() with a stronger way of generating randomness." This reverts commit 9e083fd4683294f41544e6d0d72f6e258ff3a77c. That was a few bricks shy of a load: * Query cancel stopped working * Buildfarm member pademelon stopped working, because the box doesn't have /dev/urandom nor /dev/random. This clearly needs some more discussion, and a quite different patch, so revert for now. --- contrib/pgcrypto/Makefile | 2 +- contrib/pgcrypto/internal.c | 40 ++--- contrib/pgcrypto/random.c | 247 ++++++++++++++++++++++++++++ src/backend/libpq/auth.c | 27 +-- src/backend/postmaster/postmaster.c | 153 ++++++++++++----- src/include/port.h | 3 - src/port/Makefile | 2 +- src/port/pg_strong_random.c | 148 ----------------- src/tools/msvc/Mkvcbuild.pm | 6 +- 9 files changed, 384 insertions(+), 244 deletions(-) create mode 100644 contrib/pgcrypto/random.c delete mode 100644 src/port/pg_strong_random.c diff --git a/contrib/pgcrypto/Makefile b/contrib/pgcrypto/Makefile index 76c2f1aa17..805db7626b 100644 --- a/contrib/pgcrypto/Makefile +++ b/contrib/pgcrypto/Makefile @@ -1,7 +1,7 @@ # contrib/pgcrypto/Makefile INT_SRCS = md5.c sha1.c sha2.c internal.c internal-sha2.c blf.c rijndael.c \ - fortuna.c pgp-mpi-internal.c imath.c + fortuna.c random.c pgp-mpi-internal.c imath.c INT_TESTS = sha2 OSSL_SRCS = openssl.c pgp-mpi-openssl.c diff --git a/contrib/pgcrypto/internal.c b/contrib/pgcrypto/internal.c index ad942f733a..02ff976c25 100644 --- a/contrib/pgcrypto/internal.c +++ b/contrib/pgcrypto/internal.c @@ -626,6 +626,8 @@ static time_t check_time = 0; static void system_reseed(void) { + uint8 buf[1024]; + int n; time_t t; int skip = 1; @@ -640,34 +642,24 @@ system_reseed(void) else if (check_time == 0 || (t - check_time) > SYSTEM_RESEED_CHECK_TIME) { - uint8 buf; - check_time = t; /* roll dice */ - px_get_random_bytes(&buf, 1); - skip = (buf >= SYSTEM_RESEED_CHANCE); - - /* clear 1 byte */ - px_memset(&buf, 0, sizeof(buf)); - } - if (!skip) - { - /* - * fortuna_add_entropy passes the input to SHA-256, so there's no - * point in giving it more than 256 bits of input to begin with. - */ - uint8 buf[32]; - - if (!pg_strong_random(buf, sizeof(buf))) - ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not acquire random data"))); - fortuna_add_entropy(buf, sizeof(buf)); - - seed_time = t; - px_memset(buf, 0, sizeof(buf)); + px_get_random_bytes(buf, 1); + skip = buf[0] >= SYSTEM_RESEED_CHANCE; } + /* clear 1 byte */ + px_memset(buf, 0, sizeof(buf)); + + if (skip) + return; + + n = px_acquire_system_randomness(buf); + if (n > 0) + fortuna_add_entropy(buf, n); + + seed_time = t; + px_memset(buf, 0, sizeof(buf)); } int diff --git a/contrib/pgcrypto/random.c b/contrib/pgcrypto/random.c new file mode 100644 index 0000000000..d72679e412 --- /dev/null +++ b/contrib/pgcrypto/random.c @@ -0,0 +1,247 @@ +/* + * random.c + * Acquire randomness from system. For seeding RNG. + * + * Copyright (c) 2001 Marko Kreen + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * contrib/pgcrypto/random.c + */ + +#include "postgres.h" + +#include "px.h" +#include "utils/memdebug.h" + +/* how many bytes to ask from system random provider */ +#define RND_BYTES 32 + +/* + * Try to read from /dev/urandom or /dev/random on these OS'es. + * + * The list can be pretty liberal, as the device not existing + * is expected event. + */ +#if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) \ + || defined(__NetBSD__) || defined(__DragonFly__) \ + || defined(__darwin__) || defined(__SOLARIS__) \ + || defined(__hpux) || defined(__HPUX__) \ + || defined(__CYGWIN__) || defined(_AIX) + +#define TRY_DEV_RANDOM + +#include +#include + +static int +safe_read(int fd, void *buf, size_t count) +{ + int done = 0; + char *p = buf; + int res; + + while (count) + { + res = read(fd, p, count); + if (res <= 0) + { + if (errno == EINTR) + continue; + return PXE_DEV_READ_ERROR; + } + p += res; + done += res; + count -= res; + } + return done; +} + +static uint8 * +try_dev_random(uint8 *dst) +{ + int fd; + int res; + + fd = open("/dev/urandom", O_RDONLY, 0); + if (fd == -1) + { + fd = open("/dev/random", O_RDONLY, 0); + if (fd == -1) + return dst; + } + res = safe_read(fd, dst, RND_BYTES); + close(fd); + if (res > 0) + dst += res; + return dst; +} +#endif + +/* + * Try to find randomness on Windows + */ +#ifdef WIN32 + +#define TRY_WIN32_GENRAND +#define TRY_WIN32_PERFC + +#include +#include + +/* + * this function is from libtomcrypt + * + * try to use Microsoft crypto API + */ +static uint8 * +try_win32_genrand(uint8 *dst) +{ + int res; + HCRYPTPROV h = 0; + + res = CryptAcquireContext(&h, NULL, MS_DEF_PROV, PROV_RSA_FULL, + (CRYPT_VERIFYCONTEXT | CRYPT_MACHINE_KEYSET)); + if (!res) + res = CryptAcquireContext(&h, NULL, MS_DEF_PROV, PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT | CRYPT_MACHINE_KEYSET | CRYPT_NEWKEYSET); + if (!res) + return dst; + + res = CryptGenRandom(h, RND_BYTES, dst); + if (res == TRUE) + dst += RND_BYTES; + + CryptReleaseContext(h, 0); + return dst; +} + +static uint8 * +try_win32_perfc(uint8 *dst) +{ + int res; + LARGE_INTEGER time; + + res = QueryPerformanceCounter(&time); + if (!res) + return dst; + + memcpy(dst, &time, sizeof(time)); + return dst + sizeof(time); +} +#endif /* WIN32 */ + + +/* + * If we are not on Windows, then hopefully we are + * on a unix-like system. Use the usual suspects + * for randomness. + */ +#ifndef WIN32 + +#define TRY_UNIXSTD + +#include +#include +#include +#include + +/* + * Everything here is predictible, only needs some patience. + * + * But there is a chance that the system-specific functions + * did not work. So keep faith and try to slow the attacker down. + */ +static uint8 * +try_unix_std(uint8 *dst) +{ + pid_t pid; + int x; + PX_MD *md; + struct timeval tv; + int res; + + /* process id */ + pid = getpid(); + memcpy(dst, (uint8 *) &pid, sizeof(pid)); + dst += sizeof(pid); + + /* time */ + gettimeofday(&tv, NULL); + memcpy(dst, (uint8 *) &tv, sizeof(tv)); + dst += sizeof(tv); + + /* pointless, but should not hurt */ + x = random(); + memcpy(dst, (uint8 *) &x, sizeof(x)); + dst += sizeof(x); + + /* hash of uninitialized stack and heap allocations */ + res = px_find_digest("sha1", &md); + if (res >= 0) + { + uint8 *ptr; + uint8 stack[8192]; + int alloc = 32 * 1024; + + VALGRIND_MAKE_MEM_DEFINED(stack, sizeof(stack)); + px_md_update(md, stack, sizeof(stack)); + ptr = px_alloc(alloc); + VALGRIND_MAKE_MEM_DEFINED(ptr, alloc); + px_md_update(md, ptr, alloc); + px_free(ptr); + + px_md_finish(md, dst); + px_md_free(md); + + dst += 20; + } + + return dst; +} +#endif + +/* + * try to extract some randomness for initial seeding + * + * dst should have room for 1024 bytes. + */ +unsigned +px_acquire_system_randomness(uint8 *dst) +{ + uint8 *p = dst; + +#ifdef TRY_DEV_RANDOM + p = try_dev_random(p); +#endif +#ifdef TRY_WIN32_GENRAND + p = try_win32_genrand(p); +#endif +#ifdef TRY_WIN32_PERFC + p = try_win32_perfc(p); +#endif +#ifdef TRY_UNIXSTD + p = try_unix_std(p); +#endif + return p - dst; +} diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index 44b2212b1d..0ba8530114 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -45,12 +45,6 @@ static void auth_failed(Port *port, int status, char *logdetail); static char *recv_password_packet(Port *port); static int recv_and_check_password_packet(Port *port, char **logdetail); -/*---------------------------------------------------------------- - * MD5 authentication - *---------------------------------------------------------------- - */ -static int CheckMD5Auth(Port *port, char **logdetail); - /*---------------------------------------------------------------- * Ident authentication @@ -541,7 +535,9 @@ ClientAuthentication(Port *port) ereport(FATAL, (errcode(ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION), errmsg("MD5 authentication is not supported when \"db_user_namespace\" is enabled"))); - status = CheckMD5Auth(port, &logdetail); + /* include the salt to use for computing the response */ + sendAuthRequest(port, AUTH_REQ_MD5, port->md5Salt, 4); + status = recv_and_check_password_packet(port, &logdetail); break; case uaPassword: @@ -696,25 +692,10 @@ recv_password_packet(Port *port) /*---------------------------------------------------------------- - * MD5 and password authentication + * MD5 authentication *---------------------------------------------------------------- */ -static int -CheckMD5Auth(Port *port, char **logdetail) -{ - /* include the salt to use for computing the response */ - if (!pg_strong_random(port->md5Salt, sizeof(port->md5Salt))) - { - *logdetail = psprintf(_("Could not generate random salt")); - return STATUS_ERROR; - } - - sendAuthRequest(port, AUTH_REQ_MD5, port->md5Salt, 4); - return recv_and_check_password_packet(port, logdetail); -} - - /* * Called when we have sent an authorization request for a password. * Get the response and check it. diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 4420138673..2d43506cd0 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -358,6 +358,14 @@ static volatile bool avlauncher_needs_signal = false; static volatile bool StartWorkerNeeded = true; static volatile bool HaveCrashedWorker = false; +/* + * State for assigning random salts and cancel keys. + * Also, the global MyCancelKey passes the cancel key assigned to a given + * backend from the postmaster to that backend (via fork). + */ +static unsigned int random_seed = 0; +static struct timeval random_start_time; + #ifdef USE_BONJOUR static DNSServiceRef bonjour_sdref = NULL; #endif @@ -395,6 +403,8 @@ static void processCancelRequest(Port *port, void *pkt); static int initMasks(fd_set *rmask); static void report_fork_failure_to_client(Port *port, int errnum); static CAC_state canAcceptConnections(void); +static long PostmasterRandom(void); +static void RandomSalt(char *salt, int len); static void signal_child(pid_t pid, int signal); static bool SignalSomeChildren(int signal, int targets); static void TerminateChildren(int signal); @@ -569,11 +579,9 @@ PostmasterMain(int argc, char *argv[]) * Initialize random(3) so we don't get the same values in every run. * * Note: the seed is pretty predictable from externally-visible facts such - * as postmaster start time, so don't use random() for security-critical - * random values (use pg_strong_random() instead). Backends select a - * somewhat more random seed after forking, in BackendRun(), based on the - * PID and session start timestamp, but that is still not suitable for - * security-critical values. + * as postmaster start time, so avoid using random() for security-critical + * random values during postmaster startup. At the time of first + * connection, PostmasterRandom will select a hopefully-more-random seed. */ srandom((unsigned int) (MyProcPid ^ MyStartTime)); @@ -1284,6 +1292,8 @@ PostmasterMain(int argc, char *argv[]) * Remember postmaster startup time */ PgStartTime = GetCurrentTimestamp(); + /* PostmasterRandom wants its own copy */ + gettimeofday(&random_start_time, NULL); /* * We're ready to rock and roll... @@ -2333,6 +2343,15 @@ ConnCreate(int serverFd) return NULL; } + /* + * Precompute password salt values to use for this connection. It's + * slightly annoying to do this long in advance of knowing whether we'll + * need 'em or not, but we must do the random() calls before we fork, not + * after. Else the postmaster's random sequence won't get advanced, and + * all backends would end up using the same salt... + */ + RandomSalt(port->md5Salt, sizeof(port->md5Salt)); + /* * Allocate GSSAPI specific state struct */ @@ -3885,12 +3904,7 @@ BackendStartup(Port *port) * backend will have its own copy in the forked-off process' value of * MyCancelKey, so that it can transmit the key to the frontend. */ - if (!pg_strong_random(&MyCancelKey, sizeof(MyCancelKey))) - { - ereport(LOG, - (errmsg("could not generate random query cancel key"))); - return STATUS_ERROR; - } + MyCancelKey = PostmasterRandom(); bn->cancel_key = MyCancelKey; /* Pass down canAcceptConnections state */ @@ -4198,6 +4212,13 @@ BackendRun(Port *port) int usecs; int i; + /* + * Don't want backend to be able to see the postmaster random number + * generator state. We have to clobber the static random_seed *and* start + * a new random sequence in the random() library function. + */ + random_seed = 0; + random_start_time.tv_usec = 0; /* slightly hacky way to convert timestamptz into integers */ TimestampDifference(0, port->SessionStartTime, &secs, &usecs); srandom((unsigned int) (MyProcPid ^ (usecs << 12) ^ secs)); @@ -5045,6 +5066,66 @@ StartupPacketTimeoutHandler(void) } +/* + * RandomSalt + */ +static void +RandomSalt(char *salt, int len) +{ + long rand; + int i; + + /* + * We use % 255, sacrificing one possible byte value, so as to ensure that + * all bits of the random() value participate in the result. While at it, + * add one to avoid generating any null bytes. + */ + for (i = 0; i < len; i++) + { + rand = PostmasterRandom(); + salt[i] = (rand % 255) + 1; + } +} + +/* + * PostmasterRandom + * + * Caution: use this only for values needed during connection-request + * processing. Otherwise, the intended property of having an unpredictable + * delay between random_start_time and random_stop_time will be broken. + */ +static long +PostmasterRandom(void) +{ + /* + * Select a random seed at the time of first receiving a request. + */ + if (random_seed == 0) + { + do + { + struct timeval random_stop_time; + + gettimeofday(&random_stop_time, NULL); + + /* + * We are not sure how much precision is in tv_usec, so we swap + * the high and low 16 bits of 'random_stop_time' and XOR them + * with 'random_start_time'. On the off chance that the result is + * 0, we loop until it isn't. + */ + random_seed = random_start_time.tv_usec ^ + ((random_stop_time.tv_usec << 16) | + ((random_stop_time.tv_usec >> 16) & 0xffff)); + } + while (random_seed == 0); + + srandom(random_seed); + } + + return random(); +} + /* * Count up number of child processes of specified types (dead_end chidren * are always excluded). @@ -5222,37 +5303,31 @@ StartAutovacuumWorker(void) * we'd better have something random in the field to prevent * unfriendly people from sending cancels to them. */ - if (pg_strong_random(&MyCancelKey, sizeof(MyCancelKey))) - { - bn->cancel_key = MyCancelKey; + MyCancelKey = PostmasterRandom(); + bn->cancel_key = MyCancelKey; - /* Autovac workers are not dead_end and need a child slot */ - bn->dead_end = false; - bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); - bn->bgworker_notify = false; + /* Autovac workers are not dead_end and need a child slot */ + bn->dead_end = false; + bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); + bn->bgworker_notify = false; - bn->pid = StartAutoVacWorker(); - if (bn->pid > 0) - { - bn->bkend_type = BACKEND_TYPE_AUTOVAC; - dlist_push_head(&BackendList, &bn->elem); + bn->pid = StartAutoVacWorker(); + if (bn->pid > 0) + { + bn->bkend_type = BACKEND_TYPE_AUTOVAC; + dlist_push_head(&BackendList, &bn->elem); #ifdef EXEC_BACKEND - ShmemBackendArrayAdd(bn); + ShmemBackendArrayAdd(bn); #endif - /* all OK */ - return; - } - - /* - * fork failed, fall through to report -- actual error message - * was logged by StartAutoVacWorker - */ - (void) ReleasePostmasterChildSlot(bn->child_slot); + /* all OK */ + return; } - else - ereport(LOG, - (errmsg("could not generate random query cancel key"))); + /* + * fork failed, fall through to report -- actual error message was + * logged by StartAutoVacWorker + */ + (void) ReleasePostmasterChildSlot(bn->child_slot); free(bn); } else @@ -5540,11 +5615,7 @@ assign_backendlist_entry(RegisteredBgWorker *rw) * have something random in the field to prevent unfriendly people from * sending cancels to them. */ - if (!pg_strong_random(&MyCancelKey, sizeof(MyCancelKey))) - { - rw->rw_crashed_at = GetCurrentTimestamp(); - return false; - } + MyCancelKey = PostmasterRandom(); bn->cancel_key = MyCancelKey; bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot(); diff --git a/src/include/port.h b/src/include/port.h index 4bb9feeb01..b81fa4a89e 100644 --- a/src/include/port.h +++ b/src/include/port.h @@ -454,9 +454,6 @@ extern int pg_codepage_to_encoding(UINT cp); extern char *inet_net_ntop(int af, const void *src, int bits, char *dst, size_t size); -/* port/pg_strong_random.c */ -extern bool pg_strong_random(void *buf, size_t len); - /* port/pgcheckdir.c */ extern int pg_check_dir(const char *dir); diff --git a/src/port/Makefile b/src/port/Makefile index d34f409ee9..bc9b63add0 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -32,7 +32,7 @@ LIBS += $(PTHREAD_LIBS) OBJS = $(LIBOBJS) $(PG_CRC32C_OBJS) chklocale.o erand48.o inet_net_ntop.o \ noblock.o path.o pgcheckdir.o pgmkdirp.o pgsleep.o \ - pg_strong_random.o pgstrcasecmp.o pqsignal.o \ + pgstrcasecmp.o pqsignal.o \ qsort.o qsort_arg.o quotes.o sprompt.o tar.o thread.o # foo_srv.o and foo.o are both built from foo.c, but only foo.o has -DFRONTEND diff --git a/src/port/pg_strong_random.c b/src/port/pg_strong_random.c deleted file mode 100644 index a404111d74..0000000000 --- a/src/port/pg_strong_random.c +++ /dev/null @@ -1,148 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_strong_random.c - * pg_strong_random() function to return a strong random number - * - * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group - * - * - * IDENTIFICATION - * src/port/pg_strong_random.c - * - *------------------------------------------------------------------------- - */ - -#ifndef FRONTEND -#include "postgres.h" -#else -#include "postgres_fe.h" -#endif - -#include -#include - -#ifdef USE_SSL -#include -#endif -#ifdef WIN32 -#include -#endif - -static bool random_from_file(char *filename, void *buf, size_t len); - -#ifdef WIN32 -/* - * Cache a global crypto provider that only gets freed when the process - * exits, in case we need random numbers more than once. - */ -static HCRYPTPROV hProvider = 0; -#endif - -/* - * Read (random) bytes from a file. - */ -static bool -random_from_file(char *filename, void *buf, size_t len) -{ - int f; - char *p = buf; - ssize_t res; - - f = open(filename, O_RDONLY, 0); - if (f == -1) - return false; - - while (len) - { - res = read(f, p, len); - if (res <= 0) - { - if (errno == EINTR) - continue; /* interrupted by signal, just retry */ - - close(f); - return false; - } - - p += res; - len -= res; - } - - close(f); - return true; -} - -/* - * pg_strong_random - * - * Generate requested number of random bytes. The bytes are - * cryptographically strong random, suitable for use e.g. in key - * generation. - * - * The bytes can be acquired from a number of sources, depending - * on what's available. We try the following, in this order: - * - * 1. OpenSSL's RAND_bytes() - * 2. Windows' CryptGenRandom() function - * 3. /dev/urandom - * 4. /dev/random - * - * Returns true on success, and false if none of the sources - * were available. NB: It is important to check the return value! - * Proceeding with key generation when no random data was available - * would lead to predictable keys and security issues. - */ -bool -pg_strong_random(void *buf, size_t len) -{ -#ifdef USE_SSL - - /* - * When built with OpenSSL, first try the random generation function from - * there. - */ - if (RAND_bytes(buf, len) == 1) - return true; -#endif - -#ifdef WIN32 - - /* - * Windows has CryptoAPI for strong cryptographic numbers. - */ - if (hProvider == 0) - { - if (!CryptAcquireContext(&hProvider, - NULL, - MS_DEF_PROV, - PROV_RSA_FULL, - CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) - { - /* - * On failure, set back to 0 in case the value was for some reason - * modified. - */ - hProvider = 0; - } - } - - /* Re-check in case we just retrieved the provider */ - if (hProvider != 0) - { - if (CryptGenRandom(hProvider, len, buf)) - return true; - } -#endif - - /* - * If there is no OpenSSL and no CryptoAPI (or they didn't work), then - * fall back on reading /dev/urandom or even /dev/random. - */ - if (random_from_file("/dev/urandom", buf, len)) - return true; - if (random_from_file("/dev/random", buf, len)) - return true; - - /* None of the sources were available. */ - return false; -} diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index e6c4aef99f..de764dd4d4 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -92,7 +92,7 @@ sub mkvcbuild srandom.c getaddrinfo.c gettimeofday.c inet_net_ntop.c kill.c open.c erand48.c snprintf.c strlcat.c strlcpy.c dirmod.c noblock.c path.c pgcheckdir.c pgmkdirp.c pgsleep.c pgstrcasecmp.c pqsignal.c - mkdtemp.c pg_strong_random.c qsort.c qsort_arg.c quotes.c system.c + mkdtemp.c qsort.c qsort_arg.c quotes.c system.c sprompt.c tar.c thread.c getopt.c getopt_long.c dirent.c win32env.c win32error.c win32security.c win32setlocale.c); @@ -425,8 +425,8 @@ sub mkvcbuild 'sha1.c', 'sha2.c', 'internal.c', 'internal-sha2.c', 'blf.c', 'rijndael.c', - 'fortuna.c', 'pgp-mpi-internal.c', - 'imath.c'); + 'fortuna.c', 'random.c', + 'pgp-mpi-internal.c', 'imath.c'); } $pgcrypto->AddReference($postgres); $pgcrypto->AddLibrary('ws2_32.lib'); From 6f13a682c86801cfb9ae4f3126888b42f3cb5c46 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 18 Oct 2016 12:24:46 -0400 Subject: [PATCH 337/871] Fix cidin() to handle values above 2^31 platform-independently. CommandId is declared as uint32, and values up to 4G are indeed legal. cidout() handles them properly by treating the value as unsigned int. But cidin() was just using atoi(), which has platform-dependent behavior for values outside the range of signed int, as reported by Bart Lengkeek in bug #14379. Use strtoul() instead, as xidin() does. In passing, make some purely cosmetic changes to make xidin/xidout look more like cidin/cidout; the former didn't have a monopoly on best practice IMO. Neither xidin nor cidin make any attempt to throw error for invalid input. I didn't change that here, and am not sure it's worth worrying about since neither is really a user-facing type. The point is just to ensure that indubitably-valid inputs work as expected. It's been like this for a long time, so back-patch to all supported branches. Report: <20161018152550.1413.6439@wrigleys.postgresql.org> --- src/backend/utils/adt/xid.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/backend/utils/adt/xid.c b/src/backend/utils/adt/xid.c index 9db5814219..1dad5407f9 100644 --- a/src/backend/utils/adt/xid.c +++ b/src/backend/utils/adt/xid.c @@ -41,13 +41,10 @@ Datum xidout(PG_FUNCTION_ARGS) { TransactionId transactionId = PG_GETARG_TRANSACTIONID(0); + char *result = (char *) palloc(16); - /* maximum 32 bit unsigned integer representation takes 10 chars */ - char *str = palloc(11); - - snprintf(str, 11, "%lu", (unsigned long) transactionId); - - PG_RETURN_CSTRING(str); + snprintf(result, 16, "%lu", (unsigned long) transactionId); + PG_RETURN_CSTRING(result); } /* @@ -160,12 +157,9 @@ xidComparator(const void *arg1, const void *arg2) Datum cidin(PG_FUNCTION_ARGS) { - char *s = PG_GETARG_CSTRING(0); - CommandId c; - - c = atoi(s); + char *str = PG_GETARG_CSTRING(0); - PG_RETURN_COMMANDID(c); + PG_RETURN_COMMANDID((CommandId) strtoul(str, NULL, 0)); } /* @@ -177,7 +171,7 @@ cidout(PG_FUNCTION_ARGS) CommandId c = PG_GETARG_COMMANDID(0); char *result = (char *) palloc(16); - snprintf(result, 16, "%u", (unsigned int) c); + snprintf(result, 16, "%lu", (unsigned long) c); PG_RETURN_CSTRING(result); } From fca41acb86902b90218dcc3bc0ffc462850a090f Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 18 Oct 2016 13:43:01 -0400 Subject: [PATCH 338/871] Fix typo in comment. Amit Langote --- src/include/foreign/foreign.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h index 5dc2c90f3c..143566a4ba 100644 --- a/src/include/foreign/foreign.h +++ b/src/include/foreign/foreign.h @@ -23,7 +23,7 @@ /* * Generic option types for validation. - * NB! Thes are treated as flags, so use only powers of two here. + * NB! These are treated as flags, so use only powers of two here. */ typedef enum { From 90d3da11c9417218ebd4f86b2003c98421824712 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Tue, 18 Oct 2016 10:55:56 -0700 Subject: [PATCH 339/871] Fix a few typos in simplehash.h. Author: Erik Rijkers Discussion: <274e4c8ac545d6622735f97c1f6c354b@xs4all.nl> --- src/include/lib/simplehash.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/include/lib/simplehash.h b/src/include/lib/simplehash.h index bf89c41f1b..56d1f049a4 100644 --- a/src/include/lib/simplehash.h +++ b/src/include/lib/simplehash.h @@ -256,7 +256,7 @@ SH_PREV(SH_TYPE *tb, uint32 curelem, uint32 startelem) return curelem; } -/* return distance between bucket and it's optimal position */ +/* return distance between bucket and its optimal position */ static inline uint32 SH_DISTANCE_FROM_OPTIMAL(SH_TYPE *tb, uint32 optimal, uint32 bucket) { @@ -349,7 +349,7 @@ SH_GROW(SH_TYPE *tb, uint32 newsize) * * To be able to simply move entries over, we have to start not at the * first bucket (i.e olddata[0]), but find the first bucket that's either - * empty, or is occupied by an entry at it's optimal position. Such a + * empty, or is occupied by an entry at its optimal position. Such a * bucket has to exist in any table with a load factor under 1, as not all * buckets are occupied, i.e. there always has to be an empty bucket. By * starting at such a bucket we can move the entries to the larger table, @@ -485,9 +485,9 @@ SH_INSERT(SH_TYPE *tb, SH_KEY_TYPE key, bool *found) /* * If the bucket is not empty, we either found a match (in which case * we're done), or we have to decide whether to skip over or move the - * colliding entry. When the the colliding elements distance to it's + * colliding entry. When the colliding element's distance to its * optimal position is smaller than the to-be-inserted entry's, we - * shift the colliding entry (and it's followers) forward by one. + * shift the colliding entry (and its followers) forward by one. */ if (SH_COMPARE_KEYS(tb, hash, key, entry)) From b801e120080de836b834c1b756c4c4d81ce841b5 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 18 Oct 2016 15:55:03 -0400 Subject: [PATCH 340/871] Improve regression test coverage for hash indexes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On my system, this improves coverage for src/backend/access/hash from 61.3% of lines to 88.2% of lines, and from 83.5% of functions to 97.5% of functions, which is pretty good for 36 lines of tests. Mithun Cy, reviewing by Amit Kapila and Álvaro Herrera --- src/test/regress/expected/hash_index.out | 36 ++++++++++++++++++++ src/test/regress/sql/hash_index.sql | 43 ++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/src/test/regress/expected/hash_index.out b/src/test/regress/expected/hash_index.out index 22835f8ea4..f8b9f029b2 100644 --- a/src/test/regress/expected/hash_index.out +++ b/src/test/regress/expected/hash_index.out @@ -196,3 +196,39 @@ SELECT h.seqno AS f20000 -- WHERE x = 90; -- SELECT count(*) AS i988 FROM hash_ovfl_heap -- WHERE x = 1000; +-- +-- Cause some overflow insert and splits. +-- +CREATE TABLE hash_split_heap (keycol INT); +CREATE INDEX hash_split_index on hash_split_heap USING HASH (keycol); +WARNING: hash indexes are not WAL-logged and their use is discouraged +INSERT INTO hash_split_heap SELECT 1 FROM generate_series(1, 70000) a; +VACUUM FULL hash_split_heap; +-- Let's do a backward scan. +BEGIN; +SET enable_seqscan = OFF; +SET enable_bitmapscan = OFF; +DECLARE c CURSOR FOR SELECT * from hash_split_heap WHERE keycol = 1; +MOVE FORWARD ALL FROM c; +MOVE BACKWARD 10000 FROM c; +MOVE BACKWARD ALL FROM c; +CLOSE c; +END; +-- DELETE, INSERT, REBUILD INDEX. +DELETE FROM hash_split_heap WHERE keycol = 1; +INSERT INTO hash_split_heap SELECT a/2 FROM generate_series(1, 50000) a; +VACUUM hash_split_heap; +REINDEX INDEX hash_split_index; +-- Clean up. +DROP TABLE hash_split_heap; +-- Index on temp table. +CREATE TEMP TABLE hash_temp_heap (x int, y int); +INSERT INTO hash_temp_heap VALUES (1,1); +CREATE INDEX hash_idx ON hash_temp_heap USING hash (x); +DROP TABLE hash_temp_heap CASCADE; +-- Float4 type. +CREATE TABLE hash_heap_float4 (x float4, y int); +INSERT INTO hash_heap_float4 VALUES (1.1,1); +CREATE INDEX hash_idx ON hash_heap_float4 USING hash (x); +WARNING: hash indexes are not WAL-logged and their use is discouraged +DROP TABLE hash_heap_float4 CASCADE; diff --git a/src/test/regress/sql/hash_index.sql b/src/test/regress/sql/hash_index.sql index 411e8aed39..15a3b06d17 100644 --- a/src/test/regress/sql/hash_index.sql +++ b/src/test/regress/sql/hash_index.sql @@ -151,3 +151,46 @@ SELECT h.seqno AS f20000 -- SELECT count(*) AS i988 FROM hash_ovfl_heap -- WHERE x = 1000; + +-- +-- Cause some overflow insert and splits. +-- +CREATE TABLE hash_split_heap (keycol INT); +CREATE INDEX hash_split_index on hash_split_heap USING HASH (keycol); +INSERT INTO hash_split_heap SELECT 1 FROM generate_series(1, 70000) a; + +VACUUM FULL hash_split_heap; + +-- Let's do a backward scan. +BEGIN; +SET enable_seqscan = OFF; +SET enable_bitmapscan = OFF; + +DECLARE c CURSOR FOR SELECT * from hash_split_heap WHERE keycol = 1; +MOVE FORWARD ALL FROM c; +MOVE BACKWARD 10000 FROM c; +MOVE BACKWARD ALL FROM c; +CLOSE c; +END; + +-- DELETE, INSERT, REBUILD INDEX. +DELETE FROM hash_split_heap WHERE keycol = 1; +INSERT INTO hash_split_heap SELECT a/2 FROM generate_series(1, 50000) a; + +VACUUM hash_split_heap; +REINDEX INDEX hash_split_index; + +-- Clean up. +DROP TABLE hash_split_heap; + +-- Index on temp table. +CREATE TEMP TABLE hash_temp_heap (x int, y int); +INSERT INTO hash_temp_heap VALUES (1,1); +CREATE INDEX hash_idx ON hash_temp_heap USING hash (x); +DROP TABLE hash_temp_heap CASCADE; + +-- Float4 type. +CREATE TABLE hash_heap_float4 (x float4, y int); +INSERT INTO hash_heap_float4 VALUES (1.1,1); +CREATE INDEX hash_idx ON hash_heap_float4 USING hash (x); +DROP TABLE hash_heap_float4 CASCADE; From 917dc7d2393ce680dea7a59418be9ff341df3c14 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 19 Oct 2016 14:26:05 +0300 Subject: [PATCH 341/871] Fix WAL-logging of FSM and VM truncation. When a relation is truncated, it is important that the FSM is truncated as well. Otherwise, after recovery, the FSM can return a page that has been truncated away, leading to errors like: ERROR: could not read block 28991 in file "base/16390/572026": read only 0 of 8192 bytes We were using MarkBufferDirtyHint() to dirty the buffer holding the last remaining page of the FSM, but during recovery, that might in fact not dirty the page, and the FSM update might be lost. To fix, use the stronger MarkBufferDirty() function. MarkBufferDirty() requires us to do WAL-logging ourselves, to protect from a torn page, if checksumming is enabled. Also fix an oversight in visibilitymap_truncate: it also needs to WAL-log when checksumming is enabled. Analysis by Pavan Deolasee. Discussion: --- src/backend/access/heap/visibilitymap.c | 16 ++++ src/backend/storage/freespace/freespace.c | 20 ++++- src/test/recovery/t/008_fsm_truncation.pl | 93 +++++++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 src/test/recovery/t/008_fsm_truncation.pl diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 3ad4a9f587..f020737f80 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -508,6 +508,9 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks) LockBuffer(mapBuffer, BUFFER_LOCK_EXCLUSIVE); + /* NO EREPORT(ERROR) from here till changes are logged */ + START_CRIT_SECTION(); + /* Clear out the unwanted bytes. */ MemSet(&map[truncByte + 1], 0, MAPSIZE - (truncByte + 1)); @@ -523,7 +526,20 @@ visibilitymap_truncate(Relation rel, BlockNumber nheapblocks) */ map[truncByte] &= (1 << truncOffset) - 1; + /* + * Truncation of a relation is WAL-logged at a higher-level, and we + * will be called at WAL replay. But if checksums are enabled, we need + * to still write a WAL record to protect against a torn page, if the + * page is flushed to disk before the truncation WAL record. We cannot + * use MarkBufferDirtyHint here, because that will not dirty the page + * during recovery. + */ MarkBufferDirty(mapBuffer); + if (!InRecovery && RelationNeedsWAL(rel) && XLogHintBitIsNeeded()) + log_newpage_buffer(mapBuffer, false); + + END_CRIT_SECTION(); + UnlockReleaseBuffer(mapBuffer); } else diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c index bbd90c911a..4138b04839 100644 --- a/src/backend/storage/freespace/freespace.c +++ b/src/backend/storage/freespace/freespace.c @@ -327,8 +327,26 @@ FreeSpaceMapTruncateRel(Relation rel, BlockNumber nblocks) if (!BufferIsValid(buf)) return; /* nothing to do; the FSM was already smaller */ LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + + /* NO EREPORT(ERROR) from here till changes are logged */ + START_CRIT_SECTION(); + fsm_truncate_avail(BufferGetPage(buf), first_removed_slot); - MarkBufferDirtyHint(buf, false); + + /* + * Truncation of a relation is WAL-logged at a higher-level, and we + * will be called at WAL replay. But if checksums are enabled, we need + * to still write a WAL record to protect against a torn page, if the + * page is flushed to disk before the truncation WAL record. We cannot + * use MarkBufferDirtyHint here, because that will not dirty the page + * during recovery. + */ + MarkBufferDirty(buf); + if (!InRecovery && RelationNeedsWAL(rel) && XLogHintBitIsNeeded()) + log_newpage_buffer(buf, false); + + END_CRIT_SECTION(); + UnlockReleaseBuffer(buf); new_nfsmblocks = fsm_logical_to_physical(first_removed_address) + 1; diff --git a/src/test/recovery/t/008_fsm_truncation.pl b/src/test/recovery/t/008_fsm_truncation.pl new file mode 100644 index 0000000000..9f6bdb0b64 --- /dev/null +++ b/src/test/recovery/t/008_fsm_truncation.pl @@ -0,0 +1,93 @@ +# Test WAL replay of FSM changes. +# +# FSM changes don't normally need to be WAL-logged, except for truncation. +# The FSM mustn't return a page that doesn't exist (anymore). +use strict; +use warnings; + +use PostgresNode; +use TestLib; +use Test::More tests => 1; + +my $node_master = get_new_node('master'); +$node_master->init(allows_streaming => 1); + +$node_master->append_conf('postgresql.conf', qq{ +fsync = on +wal_level = replica +wal_log_hints = on +max_prepared_transactions = 5 +autovacuum = off +}); + +# Create a master node and its standby, initializing both with some data +# at the same time. +$node_master->start; + +$node_master->backup('master_backup'); +my $node_standby = get_new_node('standby'); +$node_standby->init_from_backup($node_master, 'master_backup', + has_streaming => 1); +$node_standby->start; + +$node_master->psql('postgres', qq{ +create table testtab (a int, b char(100)); +insert into testtab select generate_series(1,1000), 'foo'; +insert into testtab select generate_series(1,1000), 'foo'; +delete from testtab where ctid > '(8,0)'; +}); + +# Take a lock on the table to prevent following vacuum from truncating it +$node_master->psql('postgres', qq{ +begin; +lock table testtab in row share mode; +prepare transaction 'p1'; +}); + +# Vacuum, update FSM without truncation +$node_master->psql('postgres', 'vacuum verbose testtab'); + +# Force a checkpoint +$node_master->psql('postgres', 'checkpoint'); + +# Now do some more insert/deletes, another vacuum to ensure full-page writes +# are done +$node_master->psql('postgres', qq{ +insert into testtab select generate_series(1,1000), 'foo'; +delete from testtab where ctid > '(8,0)'; +vacuum verbose testtab; +}); + +# Ensure all buffers are now clean on the standby +$node_standby->psql('postgres', 'checkpoint'); + +# Release the lock, vacuum again which should lead to truncation +$node_master->psql('postgres', qq{ +rollback prepared 'p1'; +vacuum verbose testtab; +}); + +$node_master->psql('postgres', 'checkpoint'); +my $until_lsn = + $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();"); + +# Wait long enough for standby to receive and apply all WAL +my $caughtup_query = + "SELECT '$until_lsn'::pg_lsn <= pg_last_xlog_replay_location()"; +$node_standby->poll_query_until('postgres', $caughtup_query) + or die "Timed out while waiting for standby to catch up"; + +# Promote the standby +$node_standby->promote; +$node_standby->poll_query_until('postgres', + "SELECT NOT pg_is_in_recovery()") + or die "Timed out while waiting for promotion of standby"; +$node_standby->psql('postgres', 'checkpoint'); + +# Restart to discard in-memory copy of FSM +$node_standby->restart; + +# Insert should work on standby +is($node_standby->psql('postgres', + qq{insert into testtab select generate_series(1,1000), 'foo';}), + 0, 'INSERT succeeds with truncated relation FSM'); From 0be22457d730da8971f761b9c948f742a12b50b2 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: [PATCH 342/871] pg_ctl: Add long options for -w and -W From: Vik Fearing --- doc/src/sgml/ref/pg_ctl-ref.sgml | 2 ++ src/bin/pg_ctl/pg_ctl.c | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index a00c355f4a..11444e85a8 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -383,6 +383,7 @@ PostgreSQL documentation + Wait for an operation to complete. This is supported for the @@ -415,6 +416,7 @@ PostgreSQL documentation + Do not wait for an operation to complete. This is the opposite of the diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index ab10d2f25c..7d97232613 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -1959,8 +1959,8 @@ do_help(void) printf(_(" -s, --silent only print errors, no informational messages\n")); printf(_(" -t, --timeout=SECS seconds to wait when using -w option\n")); printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -w wait until operation completes\n")); - printf(_(" -W do not wait until operation completes\n")); + printf(_(" -w, --wait wait until operation completes\n")); + printf(_(" -W, --no-wait do not wait until operation completes\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("(The default is to wait for shutdown, but not for start or restart.)\n\n")); printf(_("If the -D option is omitted, the environment variable PGDATA is used.\n")); @@ -2174,6 +2174,8 @@ main(int argc, char **argv) {"silent", no_argument, NULL, 's'}, {"timeout", required_argument, NULL, 't'}, {"core-files", no_argument, NULL, 'c'}, + {"wait", no_argument, NULL, 'w'}, + {"no-wait", no_argument, NULL, 'W'}, {NULL, 0, NULL, 0} }; From c709c6074083a8cc5f1ba431c741ff04e3a8a906 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: [PATCH 343/871] doc: Consistently use = sign in long options synopses This was already the predominant form in man pages and help output. --- doc/src/sgml/ref/clusterdb.sgml | 2 +- doc/src/sgml/ref/pg_ctl-ref.sgml | 6 +++--- doc/src/sgml/ref/pgupgrade.sgml | 2 +- doc/src/sgml/ref/reindexdb.sgml | 2 +- doc/src/sgml/ref/vacuumdb.sgml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/ref/clusterdb.sgml b/doc/src/sgml/ref/clusterdb.sgml index c13d74853e..67582fd6e6 100644 --- a/doc/src/sgml/ref/clusterdb.sgml +++ b/doc/src/sgml/ref/clusterdb.sgml @@ -316,7 +316,7 @@ PostgreSQL documentation foo in a database named xyzzy: -$ clusterdb --table foo xyzzy +$ clusterdb --table=foo xyzzy diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index 11444e85a8..ea0a6353d8 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -263,7 +263,7 @@ PostgreSQL documentation - + Specifies the file system location of the database configuration files. If @@ -275,7 +275,7 @@ PostgreSQL documentation - + Append the server log output to @@ -288,7 +288,7 @@ PostgreSQL documentation - + Specifies the shutdown mode. mode diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index 96851933cc..d46a730f66 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -558,7 +558,7 @@ rsync --archive --delete --hard-links --size-only old_pgdata new_pgdata remote_d run using: -psql --username postgres --file script.sql postgres +psql --username=postgres --file=script.sql postgres The scripts can be run in any order and can be deleted once they have diff --git a/doc/src/sgml/ref/reindexdb.sgml b/doc/src/sgml/ref/reindexdb.sgml index 713efc099b..36df949c95 100644 --- a/doc/src/sgml/ref/reindexdb.sgml +++ b/doc/src/sgml/ref/reindexdb.sgml @@ -396,7 +396,7 @@ PostgreSQL documentation To reindex the table foo and the index bar in a database named abcd: -$ reindexdb --table foo --index bar abcd +$ reindexdb --table=foo --index=bar abcd diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml index 92b8984b7a..4f6fa0d708 100644 --- a/doc/src/sgml/ref/vacuumdb.sgml +++ b/doc/src/sgml/ref/vacuumdb.sgml @@ -430,7 +430,7 @@ PostgreSQL documentation xyzzy, and analyze a single column bar of the table for the optimizer: -$ vacuumdb --analyze --verbose --table 'foo(bar)' xyzzy +$ vacuumdb --analyze --verbose --table='foo(bar)' xyzzy From caf936b09fc7c74844575332b07c667a178cb078 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: [PATCH 344/871] pg_ctl: Add long option for -o Now all normally used options are covered by long options as well. --- doc/src/sgml/ref/pg_ctl-ref.sgml | 2 ++ src/bin/pg_ctl/pg_ctl.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index ea0a6353d8..5fb6898699 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -301,6 +301,7 @@ PostgreSQL documentation + Specifies options to be passed directly to the @@ -316,6 +317,7 @@ PostgreSQL documentation + Specifies options to be passed directly to the diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index 7d97232613..4b476022c0 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -1972,7 +1972,7 @@ do_help(void) printf(_(" -c, --core-files not applicable on this platform\n")); #endif printf(_(" -l, --log=FILENAME write (or append) server log to FILENAME\n")); - printf(_(" -o OPTIONS command line options to pass to postgres\n" + printf(_(" -o, --options=OPTIONS command line options to pass to postgres\n" " (PostgreSQL server executable) or initdb\n")); printf(_(" -p PATH-TO-POSTGRES normally not necessary\n")); printf(_("\nOptions for stop or restart:\n")); @@ -2171,6 +2171,7 @@ main(int argc, char **argv) {"log", required_argument, NULL, 'l'}, {"mode", required_argument, NULL, 'm'}, {"pgdata", required_argument, NULL, 'D'}, + {"options", required_argument, NULL, 'o'}, {"silent", no_argument, NULL, 's'}, {"timeout", required_argument, NULL, 't'}, {"core-files", no_argument, NULL, 'c'}, From 5d58c07a441414ae29a8e315d2f9868d3d8e20be Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: [PATCH 345/871] initdb pg_basebackup: Rename --noxxx options to --no-xxx --noclean and --nosync were the only options spelled without a hyphen, so change this for consistency with other options. The options in pg_basebackup have not been in a release, so we just rename them. For initdb, we retain the old variants. Vik Fearing and me --- doc/src/sgml/ref/initdb.sgml | 4 ++-- doc/src/sgml/ref/pg_basebackup.sgml | 4 ++-- src/bin/initdb/initdb.c | 12 +++++++----- src/bin/pg_basebackup/pg_basebackup.c | 8 ++++---- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 2 +- src/test/perl/PostgresNode.pm | 2 +- src/test/regress/pg_regress.c | 2 +- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index 4e339ecce8..31f081ae7a 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -235,7 +235,7 @@ PostgreSQL documentation - + By default, initdb will wait for all files to be @@ -355,7 +355,7 @@ PostgreSQL documentation - + By default, when initdb diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 55e913f70d..7cb690dded 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -400,7 +400,7 @@ PostgreSQL documentation - + By default, when pg_basebackup aborts with an @@ -440,7 +440,7 @@ PostgreSQL documentation - + By default, pg_basebackup will wait for all files diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index e52e67df61..9e23f64130 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -2402,8 +2402,8 @@ usage(const char *progname) printf(_(" -d, --debug generate lots of debugging output\n")); printf(_(" -k, --data-checksums use data page checksums\n")); printf(_(" -L DIRECTORY where to find the input files\n")); - printf(_(" -n, --noclean do not clean up after errors\n")); - printf(_(" -N, --nosync do not wait for changes to be written safely to disk\n")); + printf(_(" -n, --no-clean do not clean up after errors\n")); + printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -s, --show show internal settings\n")); printf(_(" -S, --sync-only only sync data directory\n")); printf(_("\nOther options:\n")); @@ -3078,8 +3078,10 @@ main(int argc, char *argv[]) {"version", no_argument, NULL, 'V'}, {"debug", no_argument, NULL, 'd'}, {"show", no_argument, NULL, 's'}, - {"noclean", no_argument, NULL, 'n'}, - {"nosync", no_argument, NULL, 'N'}, + {"noclean", no_argument, NULL, 'n'}, /* for backwards compatibility */ + {"no-clean", no_argument, NULL, 'n'}, + {"nosync", no_argument, NULL, 'N'}, /* for backwards compatibility */ + {"no-sync", no_argument, NULL, 'N'}, {"sync-only", no_argument, NULL, 'S'}, {"xlogdir", required_argument, NULL, 'X'}, {"data-checksums", no_argument, NULL, 'k'}, @@ -3165,7 +3167,7 @@ main(int argc, char *argv[]) break; case 'n': noclean = true; - printf(_("Running in noclean mode. Mistakes will not be cleaned up.\n")); + printf(_("Running in no-clean mode. Mistakes will not be cleaned up.\n")); break; case 'N': do_sync = false; diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 0f5d9d6a87..76e8f449fe 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -329,8 +329,8 @@ usage(void) printf(_(" -c, --checkpoint=fast|spread\n" " set fast or spread checkpointing\n")); printf(_(" -l, --label=LABEL set backup label\n")); - printf(_(" -n, --noclean do not clean up after errors\n")); - printf(_(" -N, --nosync do not wait for changes to be written safely to disk\n")); + printf(_(" -n, --no-clean do not clean up after errors\n")); + printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -P, --progress show progress information\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -2006,8 +2006,8 @@ main(int argc, char **argv) {"gzip", no_argument, NULL, 'z'}, {"compress", required_argument, NULL, 'Z'}, {"label", required_argument, NULL, 'l'}, - {"noclean", no_argument, NULL, 'n'}, - {"nosync", no_argument, NULL, 'N'}, + {"no-clean", no_argument, NULL, 'n'}, + {"no-sync", no_argument, NULL, 'N'}, {"dbname", required_argument, NULL, 'd'}, {"host", required_argument, NULL, 'h'}, {"port", required_argument, NULL, 'p'}, diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index a52bd4e124..fcedfed2b2 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -44,7 +44,7 @@ $node->command_fails( [ 'pg_basebackup', '-D', "$tempdir/backup", '-n' ], - 'failing run with noclean option'); + 'failing run with no-clean option'); ok(-d "$tempdir/backup", 'backup directory was created and left behind'); diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 535d6c0e7c..6e5a75a050 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -484,7 +484,7 @@ sub backup print "# Taking pg_basebackup $backup_name from node \"$name\"\n"; TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-p', $port, - '-x', '--nosync'); + '-x', '--no-sync'); print "# Backup finished\n"; } diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 762adb8443..f2dedbbc8a 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -2239,7 +2239,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc /* initdb */ header(_("initializing database system")); snprintf(buf, sizeof(buf), - "\"%s%sinitdb\" -D \"%s/data\" --noclean --nosync%s%s > \"%s/log/initdb.log\" 2>&1", + "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1", bindir ? bindir : "", bindir ? "/" : "", temp_instance, From e5a9bcb529c474a07d1aa077665c5fade4c83cfc Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: [PATCH 346/871] Use pg_ctl promote -w in TAP tests Switch TAP tests to use the new wait mode of pg_ctl promote. This allows avoiding extra logic with poll_query_until() to be sure that a promoted standby is ready for read-write queries. From: Michael Paquier --- src/bin/pg_rewind/RewindTest.pm | 6 +----- src/test/perl/PostgresNode.pm | 5 +++-- src/test/recovery/t/004_timeline_switch.pl | 5 +---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/bin/pg_rewind/RewindTest.pm b/src/bin/pg_rewind/RewindTest.pm index c7c3a8f45c..1c482617ad 100644 --- a/src/bin/pg_rewind/RewindTest.pm +++ b/src/bin/pg_rewind/RewindTest.pm @@ -161,12 +161,8 @@ sub promote_standby or die "Timed out while waiting for standby to receive and write WAL"; # Now promote slave and insert some new data on master, this will put - # the master out-of-sync with the standby. Wait until the standby is - # out of recovery mode, and is ready to accept read-write connections. + # the master out-of-sync with the standby. $node_standby->promote; - $node_standby->poll_query_until('postgres', - "SELECT NOT pg_is_in_recovery()") - or die "Timed out while waiting for promotion of standby"; # Force a checkpoint after the promotion. pg_rewind looks at the control # file to determine what timeline the server is on, and that isn't updated diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 6e5a75a050..c1b16ca9e9 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -723,7 +723,7 @@ sub restart =item $node->promote() -Wrapper for pg_ctl promote +Wrapper for pg_ctl promote -w =cut @@ -735,7 +735,8 @@ sub promote my $logfile = $self->logfile; my $name = $self->name; print "### Promoting node \"$name\"\n"; - TestLib::system_log('pg_ctl', '-D', $pgdata, '-l', $logfile, 'promote'); + TestLib::system_log('pg_ctl', '-D', $pgdata, '-w', '-l', $logfile, + 'promote'); } # Internal routine to enable streaming replication on a standby node. diff --git a/src/test/recovery/t/004_timeline_switch.pl b/src/test/recovery/t/004_timeline_switch.pl index 3ee8df2cdc..5f3b2fe1dd 100644 --- a/src/test/recovery/t/004_timeline_switch.pl +++ b/src/test/recovery/t/004_timeline_switch.pl @@ -57,10 +57,7 @@ $node_standby_2->restart; # Insert some data in standby 1 and check its presence in standby 2 -# to ensure that the timeline switch has been done. Standby 1 needs -# to exit recovery first before moving on with the test. -$node_standby_1->poll_query_until('postgres', - "SELECT pg_is_in_recovery() <> true"); +# to ensure that the timeline switch has been done. $node_standby_1->safe_psql('postgres', "INSERT INTO tab_int VALUES (generate_series(1001,2000))"); $until_lsn = $node_standby_1->safe_psql('postgres', From 9ffe4a8b4cbb713bf8137f8414f02d97b6b2eb08 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: [PATCH 347/871] Make getrusage() output a little more readable Reviewed-by: Robert Haas Reviewed-by: Peter Geoghegan --- doc/src/sgml/ref/vacuum.sgml | 12 ++++++------ src/backend/tcop/postgres.c | 10 +++++----- src/backend/utils/misc/pg_rusage.c | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index dee1c5bad3..f18180a2fa 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -253,27 +253,27 @@ INFO: vacuuming "public.onek" INFO: index "onek_unique1" now contains 1000 tuples in 14 pages DETAIL: 3000 index tuples were removed. 0 index pages have been deleted, 0 are currently reusable. -CPU 0.01s/0.08u sec elapsed 0.18 sec. +CPU: user: 0.08 s, system: 0.01 s, elapsed: 0.18 s. INFO: index "onek_unique2" now contains 1000 tuples in 16 pages DETAIL: 3000 index tuples were removed. 0 index pages have been deleted, 0 are currently reusable. -CPU 0.00s/0.07u sec elapsed 0.23 sec. +CPU: user: 0.07 s, system: 0.00 s, elapsed: 0.23 s. INFO: index "onek_hundred" now contains 1000 tuples in 13 pages DETAIL: 3000 index tuples were removed. 0 index pages have been deleted, 0 are currently reusable. -CPU 0.01s/0.08u sec elapsed 0.17 sec. +CPU: user: 0.08 s, system: 0.01 s, elapsed: 0.17 s. INFO: index "onek_stringu1" now contains 1000 tuples in 48 pages DETAIL: 3000 index tuples were removed. 0 index pages have been deleted, 0 are currently reusable. -CPU 0.01s/0.09u sec elapsed 0.59 sec. +CPU: user: 0.09 s, system: 0.01 s, elapsed: 0.59 s. INFO: "onek": removed 3000 tuples in 108 pages -DETAIL: CPU 0.01s/0.06u sec elapsed 0.07 sec. +DETAIL: CPU: user: 0.06 s, system: 0.01 s, elapsed: 0.07 s. INFO: "onek": found 3000 removable, 1000 nonremovable tuples in 143 pages DETAIL: 0 dead tuples cannot be removed yet. There were 0 unused item pointers. Skipped 0 pages due to buffer pins. 0 pages are entirely empty. -CPU 0.07s/0.39u sec elapsed 1.56 sec. +CPU: user: 0.39 s, system: 0.07 s, elapsed: 1.56 s. INFO: analyzing "public.onek" INFO: "onek": 36 pages, 1000 rows sampled, 1000 estimated total rows VACUUM diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 2347f1bcdc..599874e743 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -4422,15 +4422,15 @@ ShowUsage(const char *title) appendStringInfoString(&str, "! system usage stats:\n"); appendStringInfo(&str, - "!\t%ld.%06ld elapsed %ld.%06ld user %ld.%06ld system sec\n", - (long) (elapse_t.tv_sec - Save_t.tv_sec), - (long) (elapse_t.tv_usec - Save_t.tv_usec), + "!\t%ld.%06ld s user, %ld.%06ld s system, %ld.%06ld s elapsed\n", (long) (r.ru_utime.tv_sec - Save_r.ru_utime.tv_sec), (long) (r.ru_utime.tv_usec - Save_r.ru_utime.tv_usec), (long) (r.ru_stime.tv_sec - Save_r.ru_stime.tv_sec), - (long) (r.ru_stime.tv_usec - Save_r.ru_stime.tv_usec)); + (long) (r.ru_stime.tv_usec - Save_r.ru_stime.tv_usec), + (long) (elapse_t.tv_sec - Save_t.tv_sec), + (long) (elapse_t.tv_usec - Save_t.tv_usec)); appendStringInfo(&str, - "!\t[%ld.%06ld user %ld.%06ld sys total]\n", + "!\t[%ld.%06ld s user, %ld.%06ld s system total]\n", (long) user.tv_sec, (long) user.tv_usec, (long) sys.tv_sec, diff --git a/src/backend/utils/misc/pg_rusage.c b/src/backend/utils/misc/pg_rusage.c index 8781a383c0..6602663e42 100644 --- a/src/backend/utils/misc/pg_rusage.c +++ b/src/backend/utils/misc/pg_rusage.c @@ -61,11 +61,11 @@ pg_rusage_show(const PGRUsage *ru0) } snprintf(result, sizeof(result), - "CPU %d.%02ds/%d.%02du sec elapsed %d.%02d sec", - (int) (ru1.ru.ru_stime.tv_sec - ru0->ru.ru_stime.tv_sec), - (int) (ru1.ru.ru_stime.tv_usec - ru0->ru.ru_stime.tv_usec) / 10000, + "CPU: user: %d.%02d s, system: %d.%02d s, elapsed: %d.%02d s", (int) (ru1.ru.ru_utime.tv_sec - ru0->ru.ru_utime.tv_sec), (int) (ru1.ru.ru_utime.tv_usec - ru0->ru.ru_utime.tv_usec) / 10000, + (int) (ru1.ru.ru_stime.tv_sec - ru0->ru.ru_stime.tv_sec), + (int) (ru1.ru.ru_stime.tv_usec - ru0->ru.ru_stime.tv_usec) / 10000, (int) (ru1.tv.tv_sec - ru0->tv.tv_sec), (int) (ru1.tv.tv_usec - ru0->tv.tv_usec) / 10000); From ecbac3e6e038e990f24a2e0eacdcd6738292105f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 19 Oct 2016 17:56:38 -0400 Subject: [PATCH 348/871] Update time zone data files to tzdata release 2016g. DST law changes in Turkey. Historical corrections for America/Los_Angeles, Europe/Kirov, Europe/Moscow, Europe/Samara, and Europe/Ulyanovsk. Rename Asia/Rangoon to Asia/Yangon, with a backward compatibility link. The IANA crew continue their campaign to replace invented time zone abbrevations with numeric GMT offsets. This update changes numerous zones in Antarctica and the former Soviet Union, for instance Antarctica/Casey now reports "+08" not "AWST" in the pg_timezone_names view. I kept these abbreviations in the tznames/ data files, however, so that we will still accept them for input. (We may want to start trimming those files someday, but today is not that day.) An exception is that since IANA no longer claims that "AMT" is in use in Armenia for GMT+4, I replaced it in the Default file with GMT-4, corresponding to Amazon Time which is in use in South America. It may be that that meaning is also invented and IANA will drop it in a future update; but for now, it seems silly to give pride of place to a meaning not traceable to IANA over one that is. --- src/timezone/data/africa | 2 +- src/timezone/data/antarctica | 59 +++--- src/timezone/data/asia | 152 +++++++------- src/timezone/data/australasia | 26 +-- src/timezone/data/backward | 1 + src/timezone/data/backzone | 14 +- src/timezone/data/etcetera | 74 ++++--- src/timezone/data/europe | 294 +++++++++++++++------------- src/timezone/data/factory | 9 +- src/timezone/data/northamerica | 37 +++- src/timezone/data/southamerica | 4 +- src/timezone/known_abbrevs.txt | 76 +++---- src/timezone/tznames/Antarctica.txt | 18 +- src/timezone/tznames/Asia.txt | 57 ++---- src/timezone/tznames/Default | 72 +++---- src/timezone/tznames/Europe.txt | 3 +- src/timezone/tznames/Indian.txt | 3 +- 17 files changed, 437 insertions(+), 464 deletions(-) diff --git a/src/timezone/data/africa b/src/timezone/data/africa index 50f29d5dd5..d35aaa593e 100644 --- a/src/timezone/data/africa +++ b/src/timezone/data/africa @@ -464,7 +464,7 @@ Zone Africa/Monrovia -0:43:08 - LMT 1882 # http://www.libyaherald.com/2013/10/24/correction-no-time-change-tomorrow/ # # From Paul Eggert (2013-10-25): -# For now, assume they're reverting to the pre-2012 rules of permanent UTC+2. +# For now, assume they're reverting to the pre-2012 rules of permanent UT +02. # Rule NAME FROM TO TYPE IN ON AT SAVE LETTER/S Rule Libya 1951 only - Oct 14 2:00 1:00 S diff --git a/src/timezone/data/antarctica b/src/timezone/data/antarctica index a537832954..0995835752 100644 --- a/src/timezone/data/antarctica +++ b/src/timezone/data/antarctica @@ -10,9 +10,7 @@ # http://www.spri.cam.ac.uk/bob/periant.htm # for information. # Unless otherwise specified, we have no time zone information. -# -# Except for the French entries, -# I made up all time zone abbreviations mentioned here; corrections welcome! + # FORMAT is '-00' and GMTOFF is 0 for locations while uninhabited. # Argentina - year-round bases @@ -29,7 +27,7 @@ # previously sealers and scientific personnel wintered # Margaret Turner reports # http://web.archive.org/web/20021204222245/http://www.dstc.qut.edu.au/DST/marg/daylight.html -# (1999-09-30) that they're UTC+5, with no DST; +# (1999-09-30) that they're UT +05, with no DST; # presumably this is when they have visitors. # # year-round bases @@ -68,23 +66,22 @@ # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Antarctica/Casey 0 - -00 1969 - 8:00 - AWST 2009 Oct 18 2:00 - # Australian Western Std Time - 11:00 - CAST 2010 Mar 5 2:00 # Casey Time - 8:00 - AWST 2011 Oct 28 2:00 - 11:00 - CAST 2012 Feb 21 17:00u - 8:00 - AWST + 8:00 - +08 2009 Oct 18 2:00 + 11:00 - +11 2010 Mar 5 2:00 + 8:00 - +08 2011 Oct 28 2:00 + 11:00 - +11 2012 Feb 21 17:00u + 8:00 - +08 Zone Antarctica/Davis 0 - -00 1957 Jan 13 - 7:00 - DAVT 1964 Nov # Davis Time + 7:00 - +07 1964 Nov 0 - -00 1969 Feb - 7:00 - DAVT 2009 Oct 18 2:00 - 5:00 - DAVT 2010 Mar 10 20:00u - 7:00 - DAVT 2011 Oct 28 2:00 - 5:00 - DAVT 2012 Feb 21 20:00u - 7:00 - DAVT + 7:00 - +07 2009 Oct 18 2:00 + 5:00 - +05 2010 Mar 10 20:00u + 7:00 - +07 2011 Oct 28 2:00 + 5:00 - +05 2012 Feb 21 20:00u + 7:00 - +07 Zone Antarctica/Mawson 0 - -00 1954 Feb 13 - 6:00 - MAWT 2009 Oct 18 2:00 # Mawson Time - 5:00 - MAWT + 6:00 - +06 2009 Oct 18 2:00 + 5:00 - +05 # References: # Casey Weather (1998-02-26) # http://www.antdiv.gov.au/aad/exop/sfo/casey/casey_aws.html @@ -138,7 +135,7 @@ Zone Antarctica/Mawson 0 - -00 1954 Feb 13 # # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Indian/Kerguelen 0 - -00 1950 # Port-aux-Français - 5:00 - TFT # ISO code TF Time + 5:00 - +05 # # year-round base in the main continent # Dumont d'Urville, Île des Pétrels, -6640+14001, since 1956-11 @@ -149,9 +146,9 @@ Zone Indian/Kerguelen 0 - -00 1950 # Port-aux-Français # # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Antarctica/DumontDUrville 0 - -00 1947 - 10:00 - PMT 1952 Jan 14 # Port-Martin Time + 10:00 - +10 1952 Jan 14 0 - -00 1956 Nov - 10:00 - DDUT # Dumont-d'Urville Time + 10:00 - +10 # France & Italy - year-round base # Concordia, -750600+1232000, since 2005 @@ -177,7 +174,7 @@ Zone Antarctica/DumontDUrville 0 - -00 1947 # station of Japan, it's appropriate for the principal location. # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Antarctica/Syowa 0 - -00 1957 Jan 29 - 3:00 - SYOT # Syowa Time + 3:00 - +03 # See: # NIPR Antarctic Research Activities (1999-08-17) # http://www.nipr.ac.jp/english/ara01.html @@ -214,17 +211,17 @@ Zone Antarctica/Syowa 0 - -00 1957 Jan 29 # correct, but they should be quite close to the actual dates. # # From Paul Eggert (2014-03-21): -# The CET-switching Troll rules require zic from tzcode 2014b or later, so as +# The CET-switching Troll rules require zic from tz 2014b or later, so as # suggested by Bengt-Inge Larsson comment them out for now, and approximate # with only UTC and CEST. Uncomment them when 2014b is more prevalent. # # Rule NAME FROM TO TYPE IN ON AT SAVE LETTER/S -#Rule Troll 2005 max - Mar 1 1:00u 1:00 CET -Rule Troll 2005 max - Mar lastSun 1:00u 2:00 CEST -#Rule Troll 2005 max - Oct lastSun 1:00u 1:00 CET -#Rule Troll 2004 max - Nov 7 1:00u 0:00 UTC +#Rule Troll 2005 max - Mar 1 1:00u 1:00 +01 +Rule Troll 2005 max - Mar lastSun 1:00u 2:00 +02 +#Rule Troll 2005 max - Oct lastSun 1:00u 1:00 +01 +#Rule Troll 2004 max - Nov 7 1:00u 0:00 +00 # Remove the following line when uncommenting the above '#Rule' lines. -Rule Troll 2004 max - Oct lastSun 1:00u 0:00 UTC +Rule Troll 2004 max - Oct lastSun 1:00u 0:00 +00 # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Antarctica/Troll 0 - -00 2005 Feb 12 0:00 Troll %s @@ -265,10 +262,10 @@ Zone Antarctica/Troll 0 - -00 2005 Feb 12 # changes during the year and does not necessarily correspond to mean # solar noon. So the Vostok time might have been whatever the clocks # happened to be during their visit. So we still don't really know what time -# it is at Vostok. But we'll guess UTC+6. +# it is at Vostok. But we'll guess +06. # Zone Antarctica/Vostok 0 - -00 1957 Dec 16 - 6:00 - VOST # Vostok time + 6:00 - +06 # S Africa - year-round bases # Marion Island, -4653+03752 @@ -301,7 +298,7 @@ Zone Antarctica/Vostok 0 - -00 1957 Dec 16 # # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Antarctica/Rothera 0 - -00 1976 Dec 1 - -3:00 - ROTT # Rothera time + -3:00 - -03 # Uruguay - year round base # Artigas, King George Island, -621104-0585107 diff --git a/src/timezone/data/asia b/src/timezone/data/asia index 533e2186d2..71ef8787b5 100644 --- a/src/timezone/data/asia +++ b/src/timezone/data/asia @@ -116,13 +116,11 @@ Zone Asia/Kabul 4:36:48 - LMT 1890 # http://www.worldtimezone.com/dst_news/dst_news_armenia03.html # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Asia/Yerevan 2:58:00 - LMT 1924 May 2 - 3:00 - YERT 1957 Mar # Yerevan Time - 4:00 RussiaAsia YER%sT 1991 Mar 31 2:00s - 3:00 1:00 YERST 1991 Sep 23 # independence - 3:00 RussiaAsia AM%sT 1995 Sep 24 2:00s - 4:00 - AMT 1997 - 4:00 RussiaAsia AM%sT 2012 Feb 9 - 4:00 - AMT + 3:00 - +03 1957 Mar + 4:00 RussiaAsia +04/+05 1991 Mar 31 2:00s + 3:00 RussiaAsia +03/+04 1995 Sep 24 2:00s + 4:00 - +04 1997 + 4:00 RussiaAsia +04/+05 # Azerbaijan @@ -143,13 +141,12 @@ Rule Azer 1997 2015 - Mar lastSun 4:00 1:00 S Rule Azer 1997 2015 - Oct lastSun 5:00 0 - # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Asia/Baku 3:19:24 - LMT 1924 May 2 - 3:00 - BAKT 1957 Mar # Baku Time - 4:00 RussiaAsia BAK%sT 1991 Mar 31 2:00s - 3:00 1:00 BAKST 1991 Aug 30 # independence - 3:00 RussiaAsia AZ%sT 1992 Sep lastSun 2:00s - 4:00 - AZT 1996 # Azerbaijan Time - 4:00 EUAsia AZ%sT 1997 - 4:00 Azer AZ%sT + 3:00 - +03 1957 Mar + 4:00 RussiaAsia +04/+05 1991 Mar 31 2:00s + 3:00 RussiaAsia +03/+04 1992 Sep lastSun 2:00s + 4:00 - +04 1996 + 4:00 EUAsia +04/+05 1997 + 4:00 Azer +04/+05 # Bahrain # See Asia/Qatar. @@ -268,7 +265,7 @@ Zone Asia/Brunei 7:39:40 - LMT 1926 Mar # Bandar Seri Begawan # Milne says 6:24:40 was the meridian of the time ball observatory at Rangoon. # Zone NAME GMTOFF RULES FORMAT [UNTIL] -Zone Asia/Rangoon 6:24:40 - LMT 1880 # or Yangon +Zone Asia/Yangon 6:24:40 - LMT 1880 # or Rangoon 6:24:40 - RMT 1920 # Rangoon Mean Time? 6:30 - BURT 1942 May # Burma Time 9:00 - JST 1945 May 3 @@ -383,7 +380,7 @@ Rule PRC 1987 1991 - Apr Sun>=10 0:00 1:00 D # Lewiston (ME) Daily Sun (1939-05-29), p 17, said "Even the time is # different - the occupied districts going by Tokyo time, an hour # ahead of that prevailing in the rest of Shanghai." Guess that the -# Xujiahui Observatory was under French control and stuck with UT+8. +# Xujiahui Observatory was under French control and stuck with UT +08. # # In earlier versions of this file, China had many separate Zone entries, but # this was based on what were apparently incorrect data in Shanks & Pottenger. @@ -392,26 +389,26 @@ Rule PRC 1987 1991 - Apr Sun>=10 0:00 1:00 D # Proposed in 1918 and theoretically in effect until 1949 (although in practice # mainly observed in coastal areas), the five zones were: # -# Changbai Time ("Long-white Time", Long-white = Heilongjiang area) UT+8.5 +# Changbai Time ("Long-white Time", Long-white = Heilongjiang area) UT +08:30 # Asia/Harbin (currently a link to Asia/Shanghai) # Heilongjiang (except Mohe county), Jilin # -# Zhongyuan Time ("Central plain Time") UT+8 +# Zhongyuan Time ("Central plain Time") UT +08 # Asia/Shanghai # most of China # This currently represents most other zones as well, # as apparently these regions have been the same since 1970. # Milne gives 8:05:43.2 for Xujiahui Observatory time; round to nearest. -# Guo says Shanghai switched to UT+8 "from the end of the 19th century". +# Guo says Shanghai switched to UT +08 "from the end of the 19th century". # -# Long-shu Time (probably due to Long and Shu being two names of that area) UT+7 +# Long-shu Time (probably due to Long and Shu being two names of the area) UT +07 # Asia/Chongqing (currently a link to Asia/Shanghai) # Guangxi, Guizhou, Hainan, Ningxia, Sichuan, Shaanxi, and Yunnan; # most of Gansu; west Inner Mongolia; west Qinghai; and the Guangdong # counties Deqing, Enping, Kaiping, Luoding, Taishan, Xinxing, # Yangchun, Yangjiang, Yu'nan, and Yunfu. # -# Xin-zang Time ("Xinjiang-Tibet Time") UT+6 +# Xin-zang Time ("Xinjiang-Tibet Time") UT +06 # Asia/Urumqi # This currently represents Kunlun Time as well, # as apparently the two regions have been the same since 1970. @@ -424,7 +421,7 @@ Rule PRC 1987 1991 - Apr Sun>=10 0:00 1:00 D # Shihezi, Changji, Yanqi, Heshuo, Tuokexun, Tulufan, Shanshan, Hami, # Fukang, Kuitun, Kumukuli, Miquan, Qitai, and Turfan. # -# Kunlun Time UT+5.5 +# Kunlun Time UT +05:30 # Asia/Kashgar (currently a link to Asia/Urumqi) # West Tibet, including Pulan, Aheqi, Shufu, Shule; # West Xinjiang, including Aksu, Atushi, Yining, Hetian, Cele, Luopu, Nileke, @@ -440,7 +437,7 @@ Rule PRC 1987 1991 - Apr Sun>=10 0:00 1:00 D # # On the other hand, ethnic Uyghurs, who make up about half the # population of Xinjiang, typically use "Xinjiang time" which is two -# hours behind Beijing time, or UTC +0600. The government of the Xinjiang +# hours behind Beijing time, or UT +06. The government of the Xinjiang # Uyghur Autonomous Region, (XAUR, or just Xinjiang for short) as well as # local governments such as the Ürümqi city government use both times in # publications, referring to what is popularly called Xinjiang time as @@ -496,8 +493,8 @@ Rule PRC 1987 1991 - Apr Sun>=10 0:00 1:00 D # having the same time as Beijing. # From Paul Eggert (2014-06-30): -# In the early days of the PRC, Tibet was given its own time zone (UT+6) but -# this was withdrawn in 1959 and never reinstated; see Tubten Khétsun, +# In the early days of the PRC, Tibet was given its own time zone (UT +06) +# but this was withdrawn in 1959 and never reinstated; see Tubten Khétsun, # Memories of life in Lhasa under Chinese Rule, Columbia U Press, ISBN # 978-0231142861 (2008), translator's introduction by Matthew Akester, p x. # As this is before our 1970 cutoff, Tibet doesn't need a separate zone. @@ -511,12 +508,12 @@ Rule PRC 1987 1991 - Apr Sun>=10 0:00 1:00 D # Republics, the Soviet Union, the Kuomintang, and the People's Republic of # China, and tracking down all these organizations' timekeeping rules would be # quite a trick. Approximate this lost history by a transition from LMT to -# XJT at the start of 1928, the year of accession of the warlord Jin Shuren, +# UT +06 at the start of 1928, the year of accession of the warlord Jin Shuren, # which happens to be the date given by Shanks & Pottenger (no doubt as a -# guess) as the transition from LMT. Ignore the usage of UT+8 before -# 1986-02-01 under the theory that the transition date to UT+8 is unknown and +# guess) as the transition from LMT. Ignore the usage of +08 before +# 1986-02-01 under the theory that the transition date to +08 is unknown and # that the sort of users who prefer Asia/Urumqi now typically ignored the -# UT+8 mandate back then. +# +08 mandate back then. # Zone NAME GMTOFF RULES FORMAT [UNTIL] # Beijing time, used throughout China; represented by Shanghai. @@ -721,7 +718,7 @@ Zone Asia/Hong_Kong 7:36:42 - LMT 1904 Oct 30 # be found from historical government announcement database. # From Paul Eggert (2014-07-03): -# As per Yu-Cheng Chuang, say that Taiwan was at UT+9 from 1937-10-01 +# As per Yu-Cheng Chuang, say that Taiwan was at UT +09 from 1937-10-01 # until 1945-09-21 at 01:00, overriding Shanks & Pottenger. # Likewise, use Yu-Cheng Chuang's data for DST in Taiwan. @@ -835,16 +832,15 @@ Link Asia/Nicosia Europe/Nicosia # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Asia/Tbilisi 2:59:11 - LMT 1880 2:59:11 - TBMT 1924 May 2 # Tbilisi Mean Time - 3:00 - TBIT 1957 Mar # Tbilisi Time - 4:00 RussiaAsia TBI%sT 1991 Mar 31 2:00s - 3:00 1:00 TBIST 1991 Apr 9 # independence - 3:00 RussiaAsia GE%sT 1992 # Georgia Time - 3:00 E-EurAsia GE%sT 1994 Sep lastSun - 4:00 E-EurAsia GE%sT 1996 Oct lastSun - 4:00 1:00 GEST 1997 Mar lastSun - 4:00 E-EurAsia GE%sT 2004 Jun 27 - 3:00 RussiaAsia GE%sT 2005 Mar lastSun 2:00 - 4:00 - GET + 3:00 - +03 1957 Mar + 4:00 RussiaAsia +04/+05 1991 Mar 31 2:00s + 3:00 RussiaAsia +03/+04 1992 + 3:00 E-EurAsia +03/+04 1994 Sep lastSun + 4:00 E-EurAsia +04/+05 1996 Oct lastSun + 4:00 1:00 +05 1997 Mar lastSun + 4:00 E-EurAsia +04/+05 2004 Jun 27 + 3:00 RussiaAsia +03/+04 2005 Mar lastSun 2:00 + 4:00 - +04 # East Timor @@ -921,7 +917,7 @@ Zone Asia/Kolkata 5:53:28 - LMT 1880 # Kolkata # These would be the earliest possible times for a change. # Régimes horaires pour le monde entier, by Henri Le Corre, (Éditions # Traditionnelles, 1987, Paris) says that Java and Madura switched -# from JST to UTC+07:30 on 1945-09-23, and gives 1944-09-01 for Jayapura +# from UT +09 to +07:30 on 1945-09-23, and gives 1944-09-01 for Jayapura # (Hollandia). For now, assume all Indonesian locations other than Jayapura # switched on 1945-09-23. # @@ -932,11 +928,11 @@ Zone Asia/Kolkata 5:53:28 - LMT 1880 # Kolkata # summary published by the Time and Frequency Laboratory of the # Research Center for Calibration, Instrumentation and Metrology, # Indonesia, (2006-09-29). -# The abbreviations are: +# The time zone abbreviations and UT offsets are: # -# WIB - UTC+7 - Waktu Indonesia Barat (Indonesia western time) -# WITA - UTC+8 - Waktu Indonesia Tengah (Indonesia central time) -# WIT - UTC+9 - Waktu Indonesia Timur (Indonesia eastern time) +# WIB - +07 - Waktu Indonesia Barat (Indonesia western time) +# WITA - +08 - Waktu Indonesia Tengah (Indonesia central time) +# WIT - +09 - Waktu Indonesia Timur (Indonesia eastern time) # # Zone NAME GMTOFF RULES FORMAT [UNTIL] # Java, Sumatra @@ -1825,11 +1821,11 @@ Rule Kyrgyz 1997 2005 - Mar lastSun 2:30 1:00 S Rule Kyrgyz 1997 2004 - Oct lastSun 2:30 0 - # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Asia/Bishkek 4:58:24 - LMT 1924 May 2 - 5:00 - FRUT 1930 Jun 21 # Frunze Time - 6:00 RussiaAsia FRU%sT 1991 Mar 31 2:00s - 5:00 1:00 FRUST 1991 Aug 31 2:00 # independence - 5:00 Kyrgyz KG%sT 2005 Aug 12 # Kyrgyzstan Time - 6:00 - KGT + 5:00 - +05 1930 Jun 21 + 6:00 RussiaAsia +06/+07 1991 Mar 31 2:00s + 5:00 RussiaAsia +05/+06 1991 Aug 31 2:00 + 5:00 Kyrgyz +05/+06 2005 Aug 12 + 6:00 - +06 ############################################################################### @@ -1868,25 +1864,24 @@ Rule ROK 1957 1960 - Sep Sun>=18 0:00 0 S Rule ROK 1987 1988 - May Sun>=8 2:00 1:00 D Rule ROK 1987 1988 - Oct Sun>=8 3:00 0 S -# From Paul Eggert (2014-10-30): +# From Paul Eggert (2016-08-23): # The Korean Wikipedia entry gives the following sources for UT offsets: # -# 1908: Official Journal Article No. 3994 (Edict No. 5) +# 1908: Official Journal Article No. 3994 (decree No. 5) # 1912: Governor-General of Korea Official Gazette Issue No. 367 # (Announcement No. 338) # 1954: Presidential Decree No. 876 (1954-03-17) # 1961: Law No. 676 (1961-08-07) -# 1987: Law No. 3919 (1986-12-31) # -# The Wikipedia entry also has confusing information about a change -# to UT+9 in April 1910, but then what would be the point of the later change -# to UT+9 on 1912-01-01? Omit the 1910 change for now. +# (Another source "1987: Law No. 3919 (1986-12-31)" was in the 2014-10-30 +# edition of the Korean Wikipedia entry.) # # I guessed that time zone abbreviations through 1945 followed the same # rules as discussed under Taiwan, with nominal switches from JST to KST # when the respective cities were taken over by the Allies after WWII. # -# For Pyongyang we have no information; guess no changes since World War II. +# For Pyongyang, guess no changes from World War II until 2015, as we +# have no information otherwise. # From Steffen Thorsen (2015-08-07): # According to many news sources, North Korea is going to change to @@ -2046,7 +2041,7 @@ Zone Indian/Maldives 4:54:00 - LMT 1880 # Male # Bill Bonnet (2005-05-19) reports that the US Embassy in Ulaanbaatar says # there is only one time zone and that DST is observed, citing Microsoft # Windows XP as the source. Risto Nykänen (2005-05-16) reports that -# travelmongolia.org says there are two time zones (UTC+7, UTC+8) with no DST. +# travelmongolia.org says there are two time zones (UT +07, +08) with no DST. # Oscar van Vlijmen (2005-05-20) reports that the Mongolian Embassy in # Washington, DC says there are two time zones, with DST observed. # He also found @@ -2682,7 +2677,7 @@ Link Asia/Qatar Asia/Bahrain # earlier date. # # Shanks & Pottenger also state that until 1968-05-01 Saudi Arabia had two -# time zones; the other zone, at UTC+4, was in the far eastern part of +# time zones; the other zone, at UT +04, was in the far eastern part of # the country. Ignore this, as it's before our 1970 cutoff. # # Zone NAME GMTOFF RULES FORMAT [UNTIL] @@ -2951,10 +2946,10 @@ Zone Asia/Damascus 2:25:12 - LMT 1920 # Dimashq # From Shanks & Pottenger. # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Asia/Dushanbe 4:35:12 - LMT 1924 May 2 - 5:00 - DUST 1930 Jun 21 # Dushanbe Time - 6:00 RussiaAsia DUS%sT 1991 Mar 31 2:00s - 5:00 1:00 DUSST 1991 Sep 9 2:00s - 5:00 - TJT # Tajikistan Time + 5:00 - +05 1930 Jun 21 + 6:00 RussiaAsia +06/+07 1991 Mar 31 2:00s + 5:00 1:00 +05/+06 1991 Sep 9 2:00s + 5:00 - +05 # Thailand # Zone NAME GMTOFF RULES FORMAT [UNTIL] @@ -2968,11 +2963,10 @@ Link Asia/Bangkok Asia/Vientiane # Laos # From Shanks & Pottenger. # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Asia/Ashgabat 3:53:32 - LMT 1924 May 2 # or Ashkhabad - 4:00 - ASHT 1930 Jun 21 # Ashkhabad Time - 5:00 RussiaAsia ASH%sT 1991 Mar 31 2:00 - 4:00 RussiaAsia ASH%sT 1991 Oct 27 # independence - 4:00 RussiaAsia TM%sT 1992 Jan 19 2:00 - 5:00 - TMT + 4:00 - +04 1930 Jun 21 + 5:00 RussiaAsia +05/+06 1991 Mar 31 2:00 + 4:00 RussiaAsia +04/+05 1992 Jan 19 2:00 + 5:00 - +05 # United Arab Emirates # Zone NAME GMTOFF RULES FORMAT [UNTIL] @@ -2984,20 +2978,18 @@ Link Asia/Dubai Asia/Muscat # Oman # Byalokoz 1919 says Uzbekistan was 4:27:53. # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Asia/Samarkand 4:27:53 - LMT 1924 May 2 - 4:00 - SAMT 1930 Jun 21 # Samarkand Time - 5:00 - SAMT 1981 Apr 1 - 5:00 1:00 SAMST 1981 Oct 1 - 6:00 - TAST 1982 Apr 1 # Tashkent Time - 5:00 RussiaAsia SAM%sT 1991 Sep 1 # independence - 5:00 RussiaAsia UZ%sT 1992 - 5:00 - UZT + 4:00 - +04 1930 Jun 21 + 5:00 - +05 1981 Apr 1 + 5:00 1:00 +06 1981 Oct 1 + 6:00 - +06 1982 Apr 1 + 5:00 RussiaAsia +05/+06 1992 + 5:00 - +05 # Milne says Tashkent was 4:37:10.8; round to nearest. Zone Asia/Tashkent 4:37:11 - LMT 1924 May 2 - 5:00 - TAST 1930 Jun 21 # Tashkent Time - 6:00 RussiaAsia TAS%sT 1991 Mar 31 2:00 - 5:00 RussiaAsia TAS%sT 1991 Sep 1 # independence - 5:00 RussiaAsia UZ%sT 1992 - 5:00 - UZT + 5:00 - +05 1930 Jun 21 + 6:00 RussiaAsia +06/+07 1991 Mar 31 2:00 + 5:00 RussiaAsia +05/+06 1992 + 5:00 - +05 # Vietnam diff --git a/src/timezone/data/australasia b/src/timezone/data/australasia index 0b33f67ed4..f49df1d6e1 100644 --- a/src/timezone/data/australasia +++ b/src/timezone/data/australasia @@ -545,7 +545,7 @@ Zone Pacific/Port_Moresby 9:48:40 - LMT 1880 # Base the Bougainville entry on the Arawa-Kieta region, which appears to have # the most people even though it was devastated in the Bougainville Civil War. # -# Although Shanks gives 1942-03-15 / 1943-11-01 for JST, these dates +# Although Shanks gives 1942-03-15 / 1943-11-01 for UT +09, these dates # are apparently rough guesswork from the starts of military campaigns. # The World War II entries below are instead based on Arawa-Kieta. # The Japanese occupied Kieta in July 1942, @@ -553,8 +553,8 @@ Zone Pacific/Port_Moresby 9:48:40 - LMT 1880 # http://pwencycl.kgbudge.com/B/o/Bougainville.htm # and seem to have controlled it until their 1945-08-21 surrender. # -# The Autonomous Region of Bougainville plans to switch from UTC+10 to UTC+11 -# on 2014-12-28 at 02:00. They call UTC+11 "Bougainville Standard Time"; +# The Autonomous Region of Bougainville switched from UT +10 to +11 +# on 2014-12-28 at 02:00. They call +11 "Bougainville Standard Time"; # abbreviate this as BST. See: # http://www.bougainville24.com/bougainville-issues/bougainville-gets-own-timezone/ # @@ -620,7 +620,7 @@ Link Pacific/Pago_Pago Pacific/Midway # in US minor outlying islands # From Paul Eggert (2014-06-27): # The International Date Line Act 2011 # http://www.parliament.gov.ws/images/ACTS/International_Date_Line_Act__2011_-_Eng.pdf -# changed Samoa from UTC-11 to UTC+13, effective "12 o'clock midnight, on +# changed Samoa from UT -11 to +13, effective "12 o'clock midnight, on # Thursday 29th December 2011". The International Date Line was adjusted # accordingly. @@ -715,7 +715,7 @@ Zone Pacific/Funafuti 11:56:52 - LMT 1901 # 1886-1891; Baker was similar but exact dates are not known. # Inhabited by civilians 1935-1942; U.S. military bases 1943-1944; # uninhabited thereafter. -# Howland observed Hawaii Standard Time (UT-10:30) in 1937; +# Howland observed Hawaii Standard Time (UT -10:30) in 1937; # see page 206 of Elgen M. Long and Marie K. Long, # Amelia Earhart: the Mystery Solved, Simon & Schuster (2000). # So most likely Howland and Baker observed Hawaii Time from 1935 @@ -1473,7 +1473,7 @@ Zone Pacific/Wallis 12:15:20 - LMT 1901 # Zealand time. I understand that is the time they keep locally, anyhow." # For now, assume this practice goes back to the introduction of standard time # in New Zealand, as this would make Chatham Islands time almost exactly match -# LMT back when New Zealand was at UTC+11:30; also, assume Chatham Islands did +# LMT back when New Zealand was at UT +11:30; also, assume Chatham Islands did # not observe New Zealand's prewar DST. ############################################################################### @@ -1529,7 +1529,7 @@ Zone Pacific/Wallis 12:15:20 - LMT 1901 # For now, we assume the Ladrones switched at the same time as the Philippines; # see Asia/Manila. -# US Public Law 106-564 (2000-12-23) made UTC+10 the official standard time, +# US Public Law 106-564 (2000-12-23) made UT +10 the official standard time, # under the name "Chamorro Standard Time". There is no official abbreviation, # but Congressman Robert A. Underwood, author of the bill that became law, # wrote in a press release (2000-12-27) that he will seek the use of "ChST". @@ -1541,15 +1541,15 @@ Zone Pacific/Wallis 12:15:20 - LMT 1901 # "I am certain, having lived there for the past decade, that 'Truk' # (now properly known as Chuuk) ... is in the time zone GMT+10." # -# Shanks & Pottenger write that Truk switched from UTC+10 to UTC+11 +# Shanks & Pottenger write that Truk switched from UT +10 to +11 # on 1978-10-01; ignore this for now. # From Paul Eggert (1999-10-29): # The Federated States of Micronesia Visitors Board writes in # The Federated States of Micronesia - Visitor Information (1999-01-26) # http://www.fsmgov.org/info/clocks.html -# that Truk and Yap are UTC+10, and Ponape and Kosrae are UTC+11. -# We don't know when Kosrae switched from UTC+12; assume January 1 for now. +# that Truk and Yap are UT +10, and Ponape and Kosrae are +11. +# We don't know when Kosrae switched from +12; assume January 1 for now. # Midway @@ -1615,11 +1615,11 @@ Zone Pacific/Wallis 12:15:20 - LMT 1901 # ordaining - by a masterpiece of diplomatic flattery - that # the Fourth of July should be celebrated twice in that year." -# Although Shanks & Pottenger says they both switched to UTC-11:30 -# in 1911, and to UTC-11 in 1950. many earlier sources give UTC-11 +# Although Shanks & Pottenger says they both switched to UT -11:30 +# in 1911, and to -11 in 1950. many earlier sources give -11 # for American Samoa, e.g., the US National Bureau of Standards # circular "Standard Time Throughout the World", 1932. -# Assume American Samoa switched to UTC-11 in 1911, not 1950, +# Assume American Samoa switched to -11 in 1911, not 1950, # and that after 1950 they agreed until (western) Samoa skipped a # day in 2011. Assume also that the Samoas follow the US and New # Zealand's "ST"/"DT" style of daylight-saving abbreviations. diff --git a/src/timezone/data/backward b/src/timezone/data/backward index aab237a5e2..aa23dd844e 100644 --- a/src/timezone/data/backward +++ b/src/timezone/data/backward @@ -36,6 +36,7 @@ Link Asia/Shanghai Asia/Harbin Link Asia/Urumqi Asia/Kashgar Link Asia/Kathmandu Asia/Katmandu Link Asia/Macau Asia/Macao +Link Asia/Yangon Asia/Rangoon Link Asia/Ho_Chi_Minh Asia/Saigon Link Asia/Jerusalem Asia/Tel_Aviv Link Asia/Thimphu Asia/Thimbu diff --git a/src/timezone/data/backzone b/src/timezone/data/backzone index 13dc8d4d13..4a5085f422 100644 --- a/src/timezone/data/backzone +++ b/src/timezone/data/backzone @@ -194,9 +194,9 @@ Zone Africa/Lusaka 1:53:08 - LMT 1903 Mar # Equatorial Guinea # -# Although Shanks says that Malabo switched from UTC to UTC+1 on 1963-12-15, +# Although Shanks says that Malabo switched from UT +00 to +01 on 1963-12-15, # a Google Books search says that London Calling, Issues 432-465 (1948), p 19, -# says that Spanish Guinea was at GMT+1 back then. The Shanks data entries +# says that Spanish Guinea was at +01 back then. The Shanks data entries # are most likely wrong, but we have nothing better; use them here for now. # Zone Africa/Malabo 0:35:08 - LMT 1912 @@ -479,14 +479,14 @@ Zone Asia/Muscat 3:54:24 - LMT 1920 # From Paul Eggert (2014-08-11), after a heads-up from Stephen Colebourne: # According to a Portuguese decree (1911-05-26) # http://dre.pt/pdf1sdip/1911/05/12500/23132313.pdf -# Portuguese India switched to GMT+5 on 1912-01-01. +# Portuguese India switched to UT +05 on 1912-01-01. #Zone Asia/Panaji [not enough info to complete] # Cambodia # From Paul Eggert (2014-10-11): # See Asia/Ho_Chi_Minh for the source for most of this data. Also, guess -# (1) Cambodia reverted to UT+7 on 1945-09-02, when Vietnam did, and -# (2) they also reverted to UT+7 on 1953-11-09, the date of independence. +# (1) Cambodia reverted to UT +07 on 1945-09-02, when Vietnam did, and +# (2) they also reverted to +07 on 1953-11-09, the date of independence. # These guesses are probably wrong but they're better than guessing no # transitions there. Zone Asia/Phnom_Penh 6:59:40 - LMT 1906 Jul 1 @@ -506,8 +506,8 @@ Zone Asia/Tel_Aviv 2:19:04 - LMT 1880 # Laos # From Paul Eggert (2014-10-11): # See Asia/Ho_Chi_Minh for the source for most of this data. -# Trần's book says that Laos reverted to UT+7 on 1955-04-15. -# Also, guess that Laos reverted to UT+7 on 1945-09-02, when Vietnam did; +# Trần's book says that Laos reverted to UT +07 on 1955-04-15. +# Also, guess that Laos reverted to +07 on 1945-09-02, when Vietnam did; # this is probably wrong but it's better than guessing no transition. Zone Asia/Vientiane 6:50:24 - LMT 1906 Jul 1 7:06:30 - PLMT 1911 May 1 diff --git a/src/timezone/data/etcetera b/src/timezone/data/etcetera index c2e25328d5..f5fa4c94b4 100644 --- a/src/timezone/data/etcetera +++ b/src/timezone/data/etcetera @@ -8,6 +8,13 @@ # need now for the entries that are not on UTC are for ships at sea # that cannot use POSIX TZ settings. +# Starting with POSIX 1003.1-2001, the entries below are all +# unnecessary as settings for the TZ environment variable. E.g., +# instead of TZ='Etc/GMT+4' one can use the POSIX setting TZ='<-04>+4'. +# +# Do not use a POSIX TZ setting like TZ='GMT+4', which is four hours +# behind GMT but uses the completely misleading abbreviation "GMT". + Zone Etc/GMT 0 - GMT Zone Etc/UTC 0 - UTC Zone Etc/UCT 0 - UCT @@ -26,23 +33,13 @@ Link Etc/GMT Etc/GMT-0 Link Etc/GMT Etc/GMT+0 Link Etc/GMT Etc/GMT0 -# We use POSIX-style signs in the Zone names and the output abbreviations, +# Be consistent with POSIX TZ settings in the Zone names, # even though this is the opposite of what many people expect. # POSIX has positive signs west of Greenwich, but many people expect # positive signs east of Greenwich. For example, TZ='Etc/GMT+4' uses -# the abbreviation "GMT+4" and corresponds to 4 hours behind UT +# the abbreviation "-04" and corresponds to 4 hours behind UT # (i.e. west of Greenwich) even though many people would expect it to # mean 4 hours ahead of UT (i.e. east of Greenwich). -# -# In the draft 5 of POSIX 1003.1-200x, the angle bracket notation allows for -# TZ='+4'; if you want time zone abbreviations conforming to -# ISO 8601 you can use TZ='<-0400>+4'. Thus the commonly-expected -# offset is kept within the angle bracket (and is used for display) -# while the POSIX sign is kept outside the angle bracket (and is used -# for calculation). -# -# Do not use a TZ setting like TZ='GMT+4', which is four hours behind -# GMT but uses the completely misleading abbreviation "GMT". # Earlier incarnations of this package were not POSIX-compliant, # and had lines such as @@ -51,30 +48,31 @@ Link Etc/GMT Etc/GMT0 # way does a # zic -l GMT-12 # so we moved the names into the Etc subdirectory. +# Also, the time zone abbreviations are now compatible with %z. -Zone Etc/GMT-14 14 - GMT-14 # 14 hours ahead of GMT -Zone Etc/GMT-13 13 - GMT-13 -Zone Etc/GMT-12 12 - GMT-12 -Zone Etc/GMT-11 11 - GMT-11 -Zone Etc/GMT-10 10 - GMT-10 -Zone Etc/GMT-9 9 - GMT-9 -Zone Etc/GMT-8 8 - GMT-8 -Zone Etc/GMT-7 7 - GMT-7 -Zone Etc/GMT-6 6 - GMT-6 -Zone Etc/GMT-5 5 - GMT-5 -Zone Etc/GMT-4 4 - GMT-4 -Zone Etc/GMT-3 3 - GMT-3 -Zone Etc/GMT-2 2 - GMT-2 -Zone Etc/GMT-1 1 - GMT-1 -Zone Etc/GMT+1 -1 - GMT+1 -Zone Etc/GMT+2 -2 - GMT+2 -Zone Etc/GMT+3 -3 - GMT+3 -Zone Etc/GMT+4 -4 - GMT+4 -Zone Etc/GMT+5 -5 - GMT+5 -Zone Etc/GMT+6 -6 - GMT+6 -Zone Etc/GMT+7 -7 - GMT+7 -Zone Etc/GMT+8 -8 - GMT+8 -Zone Etc/GMT+9 -9 - GMT+9 -Zone Etc/GMT+10 -10 - GMT+10 -Zone Etc/GMT+11 -11 - GMT+11 -Zone Etc/GMT+12 -12 - GMT+12 +Zone Etc/GMT-14 14 - +14 +Zone Etc/GMT-13 13 - +13 +Zone Etc/GMT-12 12 - +12 +Zone Etc/GMT-11 11 - +11 +Zone Etc/GMT-10 10 - +10 +Zone Etc/GMT-9 9 - +09 +Zone Etc/GMT-8 8 - +08 +Zone Etc/GMT-7 7 - +07 +Zone Etc/GMT-6 6 - +06 +Zone Etc/GMT-5 5 - +05 +Zone Etc/GMT-4 4 - +04 +Zone Etc/GMT-3 3 - +03 +Zone Etc/GMT-2 2 - +02 +Zone Etc/GMT-1 1 - +01 +Zone Etc/GMT+1 -1 - -01 +Zone Etc/GMT+2 -2 - -02 +Zone Etc/GMT+3 -3 - -03 +Zone Etc/GMT+4 -4 - -04 +Zone Etc/GMT+5 -5 - -05 +Zone Etc/GMT+6 -6 - -06 +Zone Etc/GMT+7 -7 - -07 +Zone Etc/GMT+8 -8 - -08 +Zone Etc/GMT+9 -9 - -09 +Zone Etc/GMT+10 -10 - -10 +Zone Etc/GMT+11 -11 - -11 +Zone Etc/GMT+12 -12 - -12 diff --git a/src/timezone/data/europe b/src/timezone/data/europe index cd3a0883d8..6020059f69 100644 --- a/src/timezone/data/europe +++ b/src/timezone/data/europe @@ -75,8 +75,7 @@ # 1:00 CET CEST CEMT Central Europe # 1:00:14 SET Swedish (1879-1899)* # 2:00 EET EEST Eastern Europe -# 3:00 FET Further-eastern Europe (2011-2014)* -# 3:00 MSK MSD MSM* Minsk, Moscow +# 3:00 MSK MSD Moscow # From Peter Ilieve (1994-12-04), # The original six [EU members]: Belgium, France, (West) Germany, Italy, @@ -583,16 +582,33 @@ Rule E-Eur 1979 1995 - Sep lastSun 0:00 0 - Rule E-Eur 1981 max - Mar lastSun 0:00 1:00 S Rule E-Eur 1996 max - Oct lastSun 0:00 0 - + +# Daylight saving time for Russia and the Soviet Union +# +# The 1917-1921 decree URLs are from Alexander Belopolsky (2016-08-23). + # Rule NAME FROM TO TYPE IN ON AT SAVE LETTER/S Rule Russia 1917 only - Jul 1 23:00 1:00 MST # Moscow Summer Time +# +# Decree No. 142 (1917-12-22) http://istmat.info/node/28137 Rule Russia 1917 only - Dec 28 0:00 0 MMT # Moscow Mean Time +# +# Decree No. 497 (1918-05-30) http://istmat.info/node/30001 Rule Russia 1918 only - May 31 22:00 2:00 MDST # Moscow Double Summer Time Rule Russia 1918 only - Sep 16 1:00 1:00 MST +# +# Decree No. 258 (1919-05-29) http://istmat.info/node/37949 Rule Russia 1919 only - May 31 23:00 2:00 MDST -Rule Russia 1919 only - Jul 1 2:00 1:00 MSD +# +Rule Russia 1919 only - Jul 1 0:00u 1:00 MSD Rule Russia 1919 only - Aug 16 0:00 0 MSK +# +# Decree No. 63 (1921-02-03) http://istmat.info/node/45840 Rule Russia 1921 only - Feb 14 23:00 1:00 MSD -Rule Russia 1921 only - Mar 20 23:00 2:00 MSM # Midsummer +# +# Decree No. 121 (1921-03-07) http://istmat.info/node/45949 +Rule Russia 1921 only - Mar 20 23:00 2:00 +05 +# Rule Russia 1921 only - Sep 1 0:00 1:00 MSD Rule Russia 1921 only - Oct 1 0:00 0 - # Act No. 925 of the Council of Ministers of the USSR (1980-10-24): @@ -775,8 +791,6 @@ Zone Europe/Vienna 1:05:21 - LMT 1893 Apr # From Alexander Bokovoy (2014-10-09): # Belarussian government decided against changing to winter time.... # http://eng.belta.by/all_news/society/Belarus-decides-against-adjusting-time-in-Russias-wake_i_76335.html -# From Paul Eggert (2014-10-08): -# Hence Belarus can share time zone abbreviations with Moscow again. # # Zone NAME GMTOFF RULES FORMAT [UNTIL] Zone Europe/Minsk 1:50:16 - LMT 1880 @@ -787,8 +801,7 @@ Zone Europe/Minsk 1:50:16 - LMT 1880 3:00 Russia MSK/MSD 1990 3:00 - MSK 1991 Mar 31 2:00s 2:00 Russia EE%sT 2011 Mar 27 2:00s - 3:00 - FET 2014 Oct 26 1:00s - 3:00 - MSK + 3:00 - +03 # Belgium # @@ -1296,7 +1309,7 @@ Zone Europe/Paris 0:09:21 - LMT 1891 Mar 15 0:01 # http://www.parlament-berlin.de/pds-fraktion.nsf/727459127c8b66ee8525662300459099/defc77cb784f180ac1256c2b0030274b/$FILE/bersarint.pdf # says that Bersarin issued an order to use Moscow time on May 20. # However, Moscow did not observe daylight saving in 1945, so -# this was equivalent to CEMT (GMT+3), not GMT+4. +# this was equivalent to UT +03, not +04. # Rule NAME FROM TO TYPE IN ON AT SAVE LETTER/S @@ -2260,7 +2273,6 @@ Zone Europe/Bucharest 1:44:24 - LMT 1891 Oct # http://www.worldtimezone.com/dst_news/dst_news_russia-map-2014-07.html # From Paul Eggert (2006-03-22): -# Except for Moscow after 1919-07-01, I invented the time zone abbreviations. # Moscow time zone abbreviations after 1919-07-01, and Moscow rules after 1991, # are from Andrey A. Chernov. The rest is from Shanks & Pottenger, # except we follow Chernov's report that 1992 DST transitions were Sat @@ -2336,7 +2348,7 @@ Zone Europe/Kaliningrad 1:22:00 - LMT 1893 Apr 2:00 Poland CE%sT 1946 3:00 Russia MSK/MSD 1989 Mar 26 2:00s 2:00 Russia EE%sT 2011 Mar 27 2:00s - 3:00 - FET 2014 Oct 26 2:00s + 3:00 - +03 2014 Oct 26 2:00s 2:00 - EET @@ -2389,6 +2401,16 @@ Zone Europe/Kaliningrad 1:22:00 - LMT 1893 Apr # 78 RU-SPE Saint Petersburg # 83 RU-NEN Nenets Autonomous Okrug +# From Paul Eggert (2016-08-23): +# The Soviets switched to UT-based time in 1919. Decree No. 59 +# (1919-02-08) http://istmat.info/node/35567 established UT-based time +# zones, and Decree No. 147 (1919-03-29) http://istmat.info/node/35854 +# specified a transition date of 1919-07-01, apparently at 00:00 UT. +# No doubt only the Soviet-controlled regions switched on that date; +# later transitions to UT-based time in other parts of Russia are +# taken from what appear to be guesses by Shanks. +# (Thanks to Alexander Belopolsky for pointers to the decrees.) + # From Stepan Golosunov (2016-03-07): # 11. Regions-violators, 1981-1982. # Wikipedia refers to @@ -2430,7 +2452,7 @@ Zone Europe/Kaliningrad 1:22:00 - LMT 1893 Apr # attributes the 1982 changes to the Act of the Council of Ministers # of the USSR No. 126 from 18.02.1982. 1980-925.txt also adds # Udmurtia to the list of affected territories and lists Khatangsky -# district separately from Taymyr Autonomous Okurg. Probably erroneously. +# district separately from Taymyr Autonomous Okrug. Probably erroneously. # # The affected territories are currently listed under Europe/Moscow, # Asia/Yekaterinburg and Asia/Krasnoyarsk. @@ -2490,7 +2512,7 @@ Zone Europe/Kaliningrad 1:22:00 - LMT 1893 Apr Zone Europe/Moscow 2:30:17 - LMT 1880 2:30:17 - MMT 1916 Jul 3 # Moscow Mean Time - 2:31:19 Russia %s 1919 Jul 1 2:00 + 2:31:19 Russia %s 1919 Jul 1 0:00u 3:00 Russia %s 1921 Oct 3:00 Russia MSK/MSD 1922 Oct 2:00 - EET 1930 Jun 21 @@ -2573,22 +2595,21 @@ Zone Europe/Astrakhan 3:12:12 - LMT 1924 May # The 1988 transition is from USSR act No. 5 (1988-01-04). Zone Europe/Volgograd 2:57:40 - LMT 1920 Jan 3 - 3:00 - TSAT 1925 Apr 6 # Tsaritsyn Time - 3:00 - STAT 1930 Jun 21 # Stalingrad Time - 4:00 - STAT 1961 Nov 11 - 4:00 Russia VOL%sT 1988 Mar 27 2:00s # Volgograd T - 3:00 Russia VOL%sT 1991 Mar 31 2:00s - 4:00 - VOLT 1992 Mar 29 2:00s - 3:00 Russia MSK/MSD 2011 Mar 27 2:00s - 4:00 - MSK 2014 Oct 26 2:00s - 3:00 - MSK + 3:00 - +03 1930 Jun 21 + 4:00 - +04 1961 Nov 11 + 4:00 Russia +04/+05 1988 Mar 27 2:00s + 3:00 Russia +03/+04 1991 Mar 31 2:00s + 4:00 - +04 1992 Mar 29 2:00s + 3:00 Russia +03/+04 2011 Mar 27 2:00s + 4:00 - +04 2014 Oct 26 2:00s + 3:00 - +03 # From Paul Eggert (2016-03-18): # Europe/Kirov covers: # 43 RU-KIR Kirov Oblast # The 1989 transition is from USSR act No. 227 (1989-03-14). # -Zone Europe/Kirov 3:18:48 - LMT 1919 Jul 1 2:00 +Zone Europe/Kirov 3:18:48 - LMT 1919 Jul 1 0:00u 3:00 - +03 1930 Jun 21 4:00 Russia +04/+05 1989 Mar 26 2:00s 3:00 Russia +03/+04 1991 Mar 31 2:00s @@ -2606,16 +2627,16 @@ Zone Europe/Kirov 3:18:48 - LMT 1919 Jul 1 2:00 # Byalokoz 1919 says Samara was 3:20:20. # The 1989 transition is from USSR act No. 227 (1989-03-14). -Zone Europe/Samara 3:20:20 - LMT 1919 Jul 1 2:00 - 3:00 - SAMT 1930 Jun 21 # Samara Time - 4:00 - SAMT 1935 Jan 27 - 4:00 Russia KUY%sT 1989 Mar 26 2:00s # Kuybyshev - 3:00 Russia MSK/MSD 1991 Mar 31 2:00s - 2:00 Russia EE%sT 1991 Sep 29 2:00s - 3:00 - SAMT 1991 Oct 20 3:00 - 4:00 Russia SAM%sT 2010 Mar 28 2:00s - 3:00 Russia SAM%sT 2011 Mar 27 2:00s - 4:00 - SAMT +Zone Europe/Samara 3:20:20 - LMT 1919 Jul 1 0:00u + 3:00 - +03 1930 Jun 21 + 4:00 - +04 1935 Jan 27 + 4:00 Russia +04/+05 1989 Mar 26 2:00s + 3:00 Russia +03/+04 1991 Mar 31 2:00s + 2:00 Russia +02/+03 1991 Sep 29 2:00s + 3:00 - +03 1991 Oct 20 3:00 + 4:00 Russia +04/+05 2010 Mar 28 2:00s + 3:00 Russia +03/+04 2011 Mar 27 2:00s + 4:00 - +04 # From Paul Eggert (2016-03-18): # Europe/Ulyanovsk covers: @@ -2630,7 +2651,7 @@ Zone Europe/Samara 3:20:20 - LMT 1919 Jul 1 2:00 # From Matt Johnson (2016-03-09): # http://publication.pravo.gov.ru/Document/View/0001201603090051 -Zone Europe/Ulyanovsk 3:13:36 - LMT 1919 Jul 1 2:00 +Zone Europe/Ulyanovsk 3:13:36 - LMT 1919 Jul 1 0:00u 3:00 - +03 1930 Jun 21 4:00 Russia +04/+05 1989 Mar 26 2:00s 3:00 Russia +03/+04 1991 Mar 31 2:00s @@ -2662,12 +2683,12 @@ Zone Europe/Ulyanovsk 3:13:36 - LMT 1919 Jul 1 2:00 Zone Asia/Yekaterinburg 4:02:33 - LMT 1916 Jul 3 3:45:05 - PMT 1919 Jul 15 4:00 - 4:00 - SVET 1930 Jun 21 # Sverdlovsk Time - 5:00 Russia SVE%sT 1991 Mar 31 2:00s - 4:00 Russia SVE%sT 1992 Jan 19 2:00s - 5:00 Russia YEK%sT 2011 Mar 27 2:00s - 6:00 - YEKT 2014 Oct 26 2:00s - 5:00 - YEKT + 4:00 - +04 1930 Jun 21 + 5:00 Russia +05/+06 1991 Mar 31 2:00s + 4:00 Russia +04/+05 1992 Jan 19 2:00s + 5:00 Russia +05/+06 2011 Mar 27 2:00s + 6:00 - +06 2014 Oct 26 2:00s + 5:00 - +05 # From Tim Parenti (2014-07-03), per Oscar van Vlijmen (2001-08-25): @@ -2677,12 +2698,12 @@ Zone Asia/Yekaterinburg 4:02:33 - LMT 1916 Jul 3 # Byalokoz 1919 says Omsk was 4:53:30. Zone Asia/Omsk 4:53:30 - LMT 1919 Nov 14 - 5:00 - OMST 1930 Jun 21 # Omsk Time - 6:00 Russia OMS%sT 1991 Mar 31 2:00s - 5:00 Russia OMS%sT 1992 Jan 19 2:00s - 6:00 Russia OMS%sT 2011 Mar 27 2:00s - 7:00 - OMST 2014 Oct 26 2:00s - 6:00 - OMST + 5:00 - +05 1930 Jun 21 + 6:00 Russia +06/+07 1991 Mar 31 2:00s + 5:00 Russia +05/+06 1992 Jan 19 2:00s + 6:00 Russia +06/+07 2011 Mar 27 2:00s + 7:00 - +07 2014 Oct 26 2:00s + 6:00 - +06 # From Paul Eggert (2016-02-22): # Asia/Barnaul covers: @@ -2762,7 +2783,7 @@ Zone Asia/Novosibirsk 5:31:40 - LMT 1919 Dec 14 6:00 # Note that time belts (numbered from 2 (Moscow) to 12 according to their # GMT/UTC offset and having too many exceptions like regions formally # belonging to one belt but using time from another) were replaced -# with time zones in 2011 with different numberings (there was a +# with time zones in 2011 with different numbering (there was a # 2-hour gap between second and third zones in 2011-2014). # From Stepan Golosunov (2016-04-12): @@ -2845,12 +2866,12 @@ Zone Asia/Novokuznetsk 5:48:48 - LMT 1924 May 1 # Byalokoz 1919 says Krasnoyarsk was 6:11:26. Zone Asia/Krasnoyarsk 6:11:26 - LMT 1920 Jan 6 - 6:00 - KRAT 1930 Jun 21 # Krasnoyarsk Time - 7:00 Russia KRA%sT 1991 Mar 31 2:00s - 6:00 Russia KRA%sT 1992 Jan 19 2:00s - 7:00 Russia KRA%sT 2011 Mar 27 2:00s - 8:00 - KRAT 2014 Oct 26 2:00s - 7:00 - KRAT + 6:00 - +06 1930 Jun 21 + 7:00 Russia +07/+08 1991 Mar 31 2:00s + 6:00 Russia +06/+07 1992 Jan 19 2:00s + 7:00 Russia +07/+08 2011 Mar 27 2:00s + 8:00 - +08 2014 Oct 26 2:00s + 7:00 - +07 # From Tim Parenti (2014-07-03), per Oscar van Vlijmen (2001-08-25): @@ -2867,12 +2888,12 @@ Zone Asia/Krasnoyarsk 6:11:26 - LMT 1920 Jan 6 Zone Asia/Irkutsk 6:57:05 - LMT 1880 6:57:05 - IMT 1920 Jan 25 # Irkutsk Mean Time - 7:00 - IRKT 1930 Jun 21 # Irkutsk Time - 8:00 Russia IRK%sT 1991 Mar 31 2:00s - 7:00 Russia IRK%sT 1992 Jan 19 2:00s - 8:00 Russia IRK%sT 2011 Mar 27 2:00s - 9:00 - IRKT 2014 Oct 26 2:00s - 8:00 - IRKT + 7:00 - +07 1930 Jun 21 + 8:00 Russia +08/+09 1991 Mar 31 2:00s + 7:00 Russia +07/+08 1992 Jan 19 2:00s + 8:00 Russia +08/+09 2011 Mar 27 2:00s + 9:00 - +09 2014 Oct 26 2:00s + 8:00 - +08 # From Tim Parenti (2014-07-06): @@ -2889,13 +2910,13 @@ Zone Asia/Irkutsk 6:57:05 - LMT 1880 # http://publication.pravo.gov.ru/Document/View/0001201512300107 Zone Asia/Chita 7:33:52 - LMT 1919 Dec 15 - 8:00 - YAKT 1930 Jun 21 # Yakutsk Time - 9:00 Russia YAK%sT 1991 Mar 31 2:00s - 8:00 Russia YAK%sT 1992 Jan 19 2:00s - 9:00 Russia YAK%sT 2011 Mar 27 2:00s - 10:00 - YAKT 2014 Oct 26 2:00s - 8:00 - IRKT 2016 Mar 27 2:00 - 9:00 - YAKT + 8:00 - +08 1930 Jun 21 + 9:00 Russia +09/+10 1991 Mar 31 2:00s + 8:00 Russia +08/+09 1992 Jan 19 2:00s + 9:00 Russia +09/+10 2011 Mar 27 2:00s + 10:00 - +10 2014 Oct 26 2:00s + 8:00 - +08 2016 Mar 27 2:00 + 9:00 - +09 # From Tim Parenti (2014-07-03), per Oscar van Vlijmen (2009-11-29): @@ -2935,12 +2956,12 @@ Zone Asia/Chita 7:33:52 - LMT 1919 Dec 15 # Byalokoz 1919 says Yakutsk was 8:38:58. Zone Asia/Yakutsk 8:38:58 - LMT 1919 Dec 15 - 8:00 - YAKT 1930 Jun 21 # Yakutsk Time - 9:00 Russia YAK%sT 1991 Mar 31 2:00s - 8:00 Russia YAK%sT 1992 Jan 19 2:00s - 9:00 Russia YAK%sT 2011 Mar 27 2:00s - 10:00 - YAKT 2014 Oct 26 2:00s - 9:00 - YAKT + 8:00 - +08 1930 Jun 21 + 9:00 Russia +09/+10 1991 Mar 31 2:00s + 8:00 Russia +08/+09 1992 Jan 19 2:00s + 9:00 Russia +09/+10 2011 Mar 27 2:00s + 10:00 - +10 2014 Oct 26 2:00s + 9:00 - +09 # From Tim Parenti (2014-07-03), per Oscar van Vlijmen (2009-11-29): @@ -2958,12 +2979,12 @@ Zone Asia/Yakutsk 8:38:58 - LMT 1919 Dec 15 # Go with Byalokoz. Zone Asia/Vladivostok 8:47:31 - LMT 1922 Nov 15 - 9:00 - VLAT 1930 Jun 21 # Vladivostok Time - 10:00 Russia VLA%sT 1991 Mar 31 2:00s - 9:00 Russia VLA%sT 1992 Jan 19 2:00s - 10:00 Russia VLA%sT 2011 Mar 27 2:00s - 11:00 - VLAT 2014 Oct 26 2:00s - 10:00 - VLAT + 9:00 - +09 1930 Jun 21 + 10:00 Russia +10/+11 1991 Mar 31 2:00s + 9:00 Russia +09/+10 1992 Jan 19 2:00s + 10:00 Russia +10/+11 2011 Mar 27 2:00s + 11:00 - +11 2014 Oct 26 2:00s + 10:00 - +10 # From Tim Parenti (2014-07-03): @@ -2981,14 +3002,14 @@ Zone Asia/Vladivostok 8:47:31 - LMT 1922 Nov 15 # This transition is no doubt wrong, but we have no better info. Zone Asia/Khandyga 9:02:13 - LMT 1919 Dec 15 - 8:00 - YAKT 1930 Jun 21 # Yakutsk Time - 9:00 Russia YAK%sT 1991 Mar 31 2:00s - 8:00 Russia YAK%sT 1992 Jan 19 2:00s - 9:00 Russia YAK%sT 2004 - 10:00 Russia VLA%sT 2011 Mar 27 2:00s - 11:00 - VLAT 2011 Sep 13 0:00s # Decree 725? - 10:00 - YAKT 2014 Oct 26 2:00s - 9:00 - YAKT + 8:00 - +08 1930 Jun 21 + 9:00 Russia +09/+10 1991 Mar 31 2:00s + 8:00 Russia +08/+09 1992 Jan 19 2:00s + 9:00 Russia +09/+10 2004 + 10:00 Russia +10/+11 2011 Mar 27 2:00s + 11:00 - +11 2011 Sep 13 0:00s # Decree 725? + 10:00 - +10 2014 Oct 26 2:00s + 9:00 - +09 # From Tim Parenti (2014-07-03): @@ -3004,15 +3025,14 @@ Zone Asia/Khandyga 9:02:13 - LMT 1919 Dec 15 # The Zone name should be Asia/Yuzhno-Sakhalinsk, but that's too long. Zone Asia/Sakhalin 9:30:48 - LMT 1905 Aug 23 - 9:00 - JCST 1937 Oct 1 - 9:00 - JST 1945 Aug 25 - 11:00 Russia SAK%sT 1991 Mar 31 2:00s # Sakhalin T - 10:00 Russia SAK%sT 1992 Jan 19 2:00s - 11:00 Russia SAK%sT 1997 Mar lastSun 2:00s - 10:00 Russia SAK%sT 2011 Mar 27 2:00s - 11:00 - SAKT 2014 Oct 26 2:00s - 10:00 - SAKT 2016 Mar 27 2:00s - 11:00 - SAKT + 9:00 - +09 1945 Aug 25 + 11:00 Russia +11/+12 1991 Mar 31 2:00s # Sakhalin T + 10:00 Russia +10/+11 1992 Jan 19 2:00s + 11:00 Russia +11/+12 1997 Mar lastSun 2:00s + 10:00 Russia +10/+11 2011 Mar 27 2:00s + 11:00 - +11 2014 Oct 26 2:00s + 10:00 - +10 2016 Mar 27 2:00s + 11:00 - +11 # From Tim Parenti (2014-07-03), per Oscar van Vlijmen (2009-11-29): @@ -3035,13 +3055,13 @@ Zone Asia/Sakhalin 9:30:48 - LMT 1905 Aug 23 # http://publication.pravo.gov.ru/Document/View/0001201604050038 Zone Asia/Magadan 10:03:12 - LMT 1924 May 2 - 10:00 - MAGT 1930 Jun 21 # Magadan Time - 11:00 Russia MAG%sT 1991 Mar 31 2:00s - 10:00 Russia MAG%sT 1992 Jan 19 2:00s - 11:00 Russia MAG%sT 2011 Mar 27 2:00s - 12:00 - MAGT 2014 Oct 26 2:00s - 10:00 - MAGT 2016 Apr 24 2:00s - 11:00 - MAGT + 10:00 - +10 1930 Jun 21 # Magadan Time + 11:00 Russia +11/+12 1991 Mar 31 2:00s + 10:00 Russia +10/+11 1992 Jan 19 2:00s + 11:00 Russia +11/+12 2011 Mar 27 2:00s + 12:00 - +12 2014 Oct 26 2:00s + 10:00 - +10 2016 Apr 24 2:00s + 11:00 - +11 # From Tim Parenti (2014-07-06): @@ -3084,17 +3104,14 @@ Zone Asia/Magadan 10:03:12 - LMT 1924 May 2 # in Russian.) In addition, Srednekolymsk appears to be a much older # settlement and the population of Zyryanka seems to be declining. # Go with Srednekolymsk. -# -# Since Magadan Oblast moves to UTC+10 on 2014-10-26, we cannot keep using MAGT -# as the abbreviation. Use SRET instead. Zone Asia/Srednekolymsk 10:14:52 - LMT 1924 May 2 - 10:00 - MAGT 1930 Jun 21 # Magadan Time - 11:00 Russia MAG%sT 1991 Mar 31 2:00s - 10:00 Russia MAG%sT 1992 Jan 19 2:00s - 11:00 Russia MAG%sT 2011 Mar 27 2:00s - 12:00 - MAGT 2014 Oct 26 2:00s - 11:00 - SRET # Srednekolymsk Time + 10:00 - +10 1930 Jun 21 + 11:00 Russia +11/+12 1991 Mar 31 2:00s + 10:00 Russia +10/+11 1992 Jan 19 2:00s + 11:00 Russia +11/+12 2011 Mar 27 2:00s + 12:00 - +12 2014 Oct 26 2:00s + 11:00 - +11 # From Tim Parenti (2014-07-03): @@ -3112,14 +3129,14 @@ Zone Asia/Srednekolymsk 10:14:52 - LMT 1924 May 2 # UTC+12 since at least then, too. Zone Asia/Ust-Nera 9:32:54 - LMT 1919 Dec 15 - 8:00 - YAKT 1930 Jun 21 # Yakutsk Time - 9:00 Russia YAKT 1981 Apr 1 - 11:00 Russia MAG%sT 1991 Mar 31 2:00s - 10:00 Russia MAG%sT 1992 Jan 19 2:00s - 11:00 Russia MAG%sT 2011 Mar 27 2:00s - 12:00 - MAGT 2011 Sep 13 0:00s # Decree 725? - 11:00 - VLAT 2014 Oct 26 2:00s - 10:00 - VLAT + 8:00 - +08 1930 Jun 21 + 9:00 Russia +09/+10 1981 Apr 1 + 11:00 Russia +11/+12 1991 Mar 31 2:00s + 10:00 Russia +10/+11 1992 Jan 19 2:00s + 11:00 Russia +11/+12 2011 Mar 27 2:00s + 12:00 - +12 2011 Sep 13 0:00s # Decree 725? + 11:00 - +11 2014 Oct 26 2:00s + 10:00 - +10 # From Tim Parenti (2014-07-03), per Oscar van Vlijmen (2001-08-25): @@ -3132,12 +3149,12 @@ Zone Asia/Ust-Nera 9:32:54 - LMT 1919 Dec 15 # The Zone name should be Asia/Petropavlovsk-Kamchatski or perhaps # Asia/Petropavlovsk-Kamchatsky, but these are too long. Zone Asia/Kamchatka 10:34:36 - LMT 1922 Nov 10 - 11:00 - PETT 1930 Jun 21 # P-K Time - 12:00 Russia PET%sT 1991 Mar 31 2:00s - 11:00 Russia PET%sT 1992 Jan 19 2:00s - 12:00 Russia PET%sT 2010 Mar 28 2:00s - 11:00 Russia PET%sT 2011 Mar 27 2:00s - 12:00 - PETT + 11:00 - +11 1930 Jun 21 + 12:00 Russia +12/+13 1991 Mar 31 2:00s + 11:00 Russia +11/+12 1992 Jan 19 2:00s + 12:00 Russia +12/+13 2010 Mar 28 2:00s + 11:00 Russia +11/+12 2011 Mar 27 2:00s + 12:00 - +12 # From Tim Parenti (2014-07-03): @@ -3145,13 +3162,13 @@ Zone Asia/Kamchatka 10:34:36 - LMT 1922 Nov 10 # 87 RU-CHU Chukotka Autonomous Okrug Zone Asia/Anadyr 11:49:56 - LMT 1924 May 2 - 12:00 - ANAT 1930 Jun 21 # Anadyr Time - 13:00 Russia ANA%sT 1982 Apr 1 0:00s - 12:00 Russia ANA%sT 1991 Mar 31 2:00s - 11:00 Russia ANA%sT 1992 Jan 19 2:00s - 12:00 Russia ANA%sT 2010 Mar 28 2:00s - 11:00 Russia ANA%sT 2011 Mar 27 2:00s - 12:00 - ANAT + 12:00 - +12 1930 Jun 21 + 13:00 Russia +13/+14 1982 Apr 1 0:00s + 12:00 Russia +12/+13 1991 Mar 31 2:00s + 11:00 Russia +11/+12 1992 Jan 19 2:00s + 12:00 Russia +12/+13 2010 Mar 28 2:00s + 11:00 Russia +11/+12 2011 Mar 27 2:00s + 12:00 - +12 # San Marino @@ -3472,6 +3489,14 @@ Zone Europe/Zurich 0:34:08 - LMT 1853 Jul 16 # See above comment. # Engineered Standard Time," said Twitter user @aysekarahasan. # http://www.bbc.com/news/world-europe-34631326 +# From Burak AYDIN (2016-09-08): +# Turkey will stay in Daylight Saving Time even in winter.... +# http://www.resmigazete.gov.tr/eskiler/2016/09/20160908-2.pdf +# +# From Paul Eggert (2016-09-07): +# The change is permanent, so this is the new standard time in Turkey. +# It takes effect today, which is not much notice. + # Rule NAME FROM TO TYPE IN ON AT SAVE LETTER/S Rule Turkey 1916 only - May 1 0:00 1:00 S Rule Turkey 1916 only - Oct 1 0:00 0 - @@ -3535,7 +3560,7 @@ Rule Turkey 1996 2006 - Oct lastSun 1:00s 0 - Zone Europe/Istanbul 1:55:52 - LMT 1880 1:56:56 - IMT 1910 Oct # Istanbul Mean Time? 2:00 Turkey EE%sT 1978 Oct 15 - 3:00 Turkey TR%sT 1985 Apr 20 # Turkey Time + 3:00 Turkey +03/+04 1985 Apr 20 2:00 Turkey EE%sT 2007 2:00 EU EE%sT 2011 Mar 27 1:00u 2:00 - EET 2011 Mar 28 1:00u @@ -3543,7 +3568,8 @@ Zone Europe/Istanbul 1:55:52 - LMT 1880 2:00 - EET 2014 Mar 31 1:00u 2:00 EU EE%sT 2015 Oct 25 1:00u 2:00 1:00 EEST 2015 Nov 8 1:00u - 2:00 EU EE%sT + 2:00 EU EE%sT 2016 Sep 7 + 3:00 - +03 Link Europe/Istanbul Asia/Istanbul # Istanbul is in both continents. # Ukraine diff --git a/src/timezone/data/factory b/src/timezone/data/factory index 4304f7cf7b..75fa4a11c3 100644 --- a/src/timezone/data/factory +++ b/src/timezone/data/factory @@ -1,9 +1,10 @@ # This file is in the public domain, so clarified as of # 2009-05-17 by Arthur David Olson. -# For companies who don't want to put time zone specification in -# their installation procedures. When users run date, they'll get the message. -# Also useful for the "comp.sources" version. +# For distributors who don't want to put time zone specification in +# their installation procedures. Users that run 'date' will get the +# time zone abbreviation "-00", indicating that the actual time zone +# is unknown. # Zone NAME GMTOFF RULES FORMAT -Zone Factory 0 - "Local time zone must be set--see zic manual page" +Zone Factory 0 - -00 diff --git a/src/timezone/data/northamerica b/src/timezone/data/northamerica index 6256f970a8..0bafb00a22 100644 --- a/src/timezone/data/northamerica +++ b/src/timezone/data/northamerica @@ -413,11 +413,42 @@ Zone America/Denver -6:59:56 - LMT 1883 Nov 18 12:00:04 # north of the Salmon River, and the towns of Burgdorf and Warren), # Nevada (except West Wendover), Oregon (except the northern 3/4 of # Malheur county), and Washington + +# From Paul Eggert (2016-08-20): +# In early February 1948, in response to California's electricity shortage, +# PG&E changed power frequency from 60 to 59.5 Hz during daylight hours, +# causing electric clocks to lose six minutes per day. (This did not change +# legal time, and is not part of the data here.) See: +# Ross SA. An energy crisis from the past: Northern California in 1948. +# Working Paper No. 8, Institute of Governmental Studies, UC Berkeley, +# 1973-11. http://escholarship.org/uc/item/8x22k30c +# +# In another measure to save electricity, DST was instituted from 1948-03-14 +# at 02:01 to 1949-01-16 at 02:00, with the governor having the option to move +# the fallback transition earlier. See pages 3-4 of: +# http://clerk.assembly.ca.gov/sites/clerk.assembly.ca.gov/files/archive/Statutes/1948/48Vol1_Chapters.pdf +# +# In response: +# +# Governor Warren received a torrent of objecting mail, and it is not too much +# to speculate that the objections to Daylight Saving Time were one important +# factor in the defeat of the Dewey-Warren Presidential ticket in California. +# -- Ross, p 25 +# +# On December 8 the governor exercised the option, setting the date to January 1 +# (LA Times 1948-12-09). The transition time was 02:00 (LA Times 1949-01-01). +# +# Despite the controversy, in 1949 California voters approved Proposition 12, +# which established DST from April's last Sunday at 01:00 until September's +# last Sunday at 02:00. This was amended by 1962's Proposition 6, which changed +# the fall-back date to October's last Sunday. See: +# http://repository.uchastings.edu/cgi/viewcontent.cgi?article=1501&context=ca_ballot_props +# http://repository.uchastings.edu/cgi/viewcontent.cgi?article=1636&context=ca_ballot_props # # Rule NAME FROM TO TYPE IN ON AT SAVE LETTER -Rule CA 1948 only - Mar 14 2:00 1:00 D +Rule CA 1948 only - Mar 14 2:01 1:00 D Rule CA 1949 only - Jan 1 2:00 0 S -Rule CA 1950 1966 - Apr lastSun 2:00 1:00 D +Rule CA 1950 1966 - Apr lastSun 1:00 1:00 D Rule CA 1950 1961 - Sep lastSun 2:00 0 S Rule CA 1962 1966 - Oct lastSun 2:00 0 S # Zone NAME GMTOFF RULES FORMAT [UNTIL] @@ -3281,7 +3312,7 @@ Zone America/Miquelon -3:44:40 - LMT 1911 May 15 # St Pierre # indicating that the normal ET rules are followed. # # From Paul Eggert (2014-08-19): -# The 2014-08-13 Cabinet meeting decided to stay on UTC-4 year-round. See: +# The 2014-08-13 Cabinet meeting decided to stay on UT -04 year-round. See: # http://tcweeklynews.com/daylight-savings-time-to-be-maintained-p5353-127.htm # Model this as a switch from EST/EDT to AST ... # From Chris Walton (2014-11-04): diff --git a/src/timezone/data/southamerica b/src/timezone/data/southamerica index 1c38f63d1c..532145172f 100644 --- a/src/timezone/data/southamerica +++ b/src/timezone/data/southamerica @@ -410,9 +410,9 @@ Rule Arg 2008 only - Oct Sun>=15 0:00 1:00 S # stuck on Summer daylight savings time even though the summer is over. # From Paul Eggert (2013-09-05): -# Perhaps San Luis operates on the legal fiction that it is at UTC-4 +# Perhaps San Luis operates on the legal fiction that it is at -04 # with perpetual summer time, but ordinary usage typically seems to -# just say it's at UTC-3; see, for example, +# just say it's at -03; see, for example, # http://es.wikipedia.org/wiki/Hora_oficial_argentina # We've documented similar situations as being plain changes to # standard time, so let's do that here too. This does not change UTC diff --git a/src/timezone/known_abbrevs.txt b/src/timezone/known_abbrevs.txt index 015f4c90ba..9c2cd9dcc7 100644 --- a/src/timezone/known_abbrevs.txt +++ b/src/timezone/known_abbrevs.txt @@ -1,8 +1,32 @@ ++00 0 ++01 3600 ++02 7200 ++02 7200 D +03 10800 +04 14400 +05 18000 +06 21600 +07 25200 ++08 28800 ++09 32400 ++10 36000 ++11 39600 ++12 43200 ++13 46800 ++14 50400 +-00 0 +-01 -3600 +-02 -7200 +-03 -10800 +-04 -14400 +-05 -18000 +-06 -21600 +-07 -25200 +-08 -28800 +-09 -32400 +-10 -36000 +-11 -39600 +-12 -43200 ACDT 37800 D ACST 34200 ACT -18000 @@ -15,15 +39,12 @@ AKDT -28800 D AKST -32400 AMST -10800 D AMT -14400 -AMT 14400 -ANAT 43200 ART -10800 AST -14400 AST 10800 AWST 28800 AZOST 0 D AZOT -3600 -AZT 14400 BDT 21600 BNT 28800 BOT -14400 @@ -53,8 +74,6 @@ CST 28800 CVT -3600 CXT 25200 ChST 36000 -DAVT 25200 -DDUT 36000 EASST -18000 D EAST -21600 EAT 10800 @@ -71,36 +90,9 @@ FKST -10800 FNT -7200 GALT -21600 GAMT -32400 -GET 14400 GFT -10800 GILT 43200 GMT 0 -GMT+1 -3600 -GMT+10 -36000 -GMT+11 -39600 -GMT+12 -43200 -GMT+2 -7200 -GMT+3 -10800 -GMT+4 -14400 -GMT+5 -18000 -GMT+6 -21600 -GMT+7 -25200 -GMT+8 -28800 -GMT+9 -32400 -GMT-1 3600 -GMT-10 36000 -GMT-11 39600 -GMT-12 43200 -GMT-13 46800 -GMT-14 50400 -GMT-2 7200 -GMT-3 10800 -GMT-4 14400 -GMT-5 18000 -GMT-6 21600 -GMT-7 25200 -GMT-8 28800 -GMT-9 32400 GST -7200 GST 14400 GYT -14400 @@ -113,23 +105,18 @@ ICT 25200 IDT 10800 D IOT 21600 IRDT 16200 D -IRKT 28800 IRST 12600 IST 19800 IST 3600 D IST 7200 JST 32400 -KGT 21600 KOST 39600 -KRAT 25200 KST 30600 KST 32400 LHDT 39600 D LHST 37800 LINT 50400 -MAGT 39600 MART -34200 -MAWT 18000 MDT -21600 D MEST 7200 D MET 3600 @@ -150,10 +137,8 @@ NST -12600 NUT -39600 NZDT 46800 D NZST 43200 -OMST 21600 PDT -25200 D PET -18000 -PETT 43200 PGT 36000 PHOT 46800 PHT 28800 @@ -166,23 +151,15 @@ PWT 32400 PYST -10800 D PYT -14400 RET 14400 -ROTT -10800 -SAKT 39600 -SAMT 14400 SAST 7200 SBT 39600 SCT 14400 SGT 28800 -SRET 39600 SRT -10800 SST -39600 -SYOT 10800 TAHT -36000 -TFT 18000 -TJT 18000 TKT 46800 TLT 32400 -TMT 18000 TOT 46800 TVT 43200 UCT 0 @@ -190,10 +167,7 @@ ULAST 32400 D ULAT 28800 UTC 0 UYT -10800 -UZT 18000 VET -14400 -VLAT 36000 -VOST 21600 VUT 39600 WAKT 43200 WAST 7200 D @@ -209,5 +183,3 @@ WITA 28800 WSDT 50400 D WSST 46800 XJT 21600 -YAKT 32400 -YEKT 18000 diff --git a/src/timezone/tznames/Antarctica.txt b/src/timezone/tznames/Antarctica.txt index 942e81993a..1a0729dcbb 100644 --- a/src/timezone/tznames/Antarctica.txt +++ b/src/timezone/tznames/Antarctica.txt @@ -16,12 +16,9 @@ CLST -10800 D # Chile Summer Time CLT America/Santiago # Chile Time # (America/Santiago) # (Antarctica/Palmer) -DAVT Antarctica/Davis # Davis Time (Antarctica) - # (Antarctica/Davis) -DDUT 36000 # Dumont-d`Urville Time (Antarctica) - # (Antarctica/DumontDUrville) -MAWT Antarctica/Mawson # Mawson Time (Antarctica) - # (Antarctica/Mawson) +DAVT Antarctica/Davis # Davis Time (Antarctica) (obsolete) +DDUT 36000 # Dumont-d`Urville Time (Antarctica) (obsolete) +MAWT Antarctica/Mawson # Mawson Time (Antarctica) (obsolete) MIST 39600 # Macquarie Island Time # (Antarctica/Macquarie) NZDT 46800 D # New Zealand Daylight Time @@ -30,9 +27,6 @@ NZDT 46800 D # New Zealand Daylight Time NZST 43200 # New Zealand Standard Time # (Antarctica/McMurdo) # (Pacific/Auckland) -ROTT -10800 # Rothera Time - # (Antarctica/Rothera) -SYOT 10800 # Syowa Time - # (Antarctica/Syowa) -VOST 21600 # Vostok time - # (Antarctica/Vostok) +ROTT -10800 # Rothera Time (obsolete) +SYOT 10800 # Syowa Time (obsolete) +VOST 21600 # Vostok time (obsolete) diff --git a/src/timezone/tznames/Asia.txt b/src/timezone/tznames/Asia.txt index f17dbfe9f2..c834b6f1e4 100644 --- a/src/timezone/tznames/Asia.txt +++ b/src/timezone/tznames/Asia.txt @@ -19,11 +19,9 @@ AMST Asia/Yerevan # Armenia Summer Time # CONFLICT! AMT is not unique # Other timezones: # - AMT: Amazon Time (America) -AMT Asia/Yerevan # Armenia Time - # (Asia/Yerevan) +AMT Asia/Yerevan # Armenia Time (obsolete) ANAST Asia/Anadyr # Anadyr Summer Time (obsolete) -ANAT Asia/Anadyr # Anadyr Time - # (Asia/Anadyr) +ANAT Asia/Anadyr # Anadyr Time (obsolete) AQTST Asia/Aqtau # Aqtau Summer Time (obsolete) AQTT Asia/Aqtau # Aqtau Time (obsolete) # CONFLICT! AST is not unique @@ -42,8 +40,7 @@ AST 10800 # Arabia Standard Time # (Asia/Qatar) # (Asia/Riyadh) AZST Asia/Baku # Azerbaijan Summer Time (obsolete) -AZT Asia/Baku # Azerbaijan Time - # (Asia/Baku) +AZT Asia/Baku # Azerbaijan Time (obsolete) BDT 21600 # Bangladesh Time # (Asia/Dhaka) BNT 28800 # Brunei Darussalam Time @@ -115,8 +112,7 @@ EET 7200 # East-Egypt Time # (Europe/Zaporozhye) EIT 32400 # East Indonesia Time (obsolete, WIT is now preferred) GEST Asia/Tbilisi # Georgia Summer Time (obsolete) -GET Asia/Tbilisi # Georgia Time - # (Asia/Tbilisi) +GET Asia/Tbilisi # Georgia Time (obsolete) # CONFLICT! GST is not unique # Other timezones: # - GST: South Georgia Time (Atlantic) @@ -138,8 +134,7 @@ IDT 10800 D # Israel Daylight Time IRDT Asia/Tehran # Iran Daylight Time # (Asia/Tehran) IRKST Asia/Irkutsk # Irkutsk Summer Time (obsolete) -IRKT Asia/Irkutsk # Irkutsk Time - # (Asia/Irkutsk) +IRKT Asia/Irkutsk # Irkutsk Time (obsolete) IRST Asia/Tehran # Iran Standard Time # (Asia/Tehran) IRT 12600 # Iran Time (not in IANA database) @@ -160,21 +155,18 @@ JST 32400 # Japan Standard Time # (Asia/Tokyo) KDT 36000 D # Korean Daylight Time (not in IANA database) KGST 21600 D # Kyrgyzstan Summer Time (obsolete) -KGT Asia/Bishkek # Kyrgyzstan Time - # (Asia/Bishkek) +KGT Asia/Bishkek # Kyrgyzstan Time (obsolete) KRAST Asia/Krasnoyarsk # Krasnoyarsk Summer Time (obsolete) -KRAT Asia/Krasnoyarsk # Krasnoyarsk Time - # (Asia/Krasnoyarsk) +KRAT Asia/Krasnoyarsk # Krasnoyarsk Time (obsolete) KST Asia/Pyongyang # Korean Standard Time # (Asia/Pyongyang) KST 32400 # Korean Standard Time # (Asia/Seoul) LKT Asia/Colombo # Lanka Time (obsolete) MAGST Asia/Magadan # Magadan Summer Time (obsolete) -MAGT Asia/Magadan # Magadan Time - # (Asia/Magadan) +MAGT Asia/Magadan # Magadan Time (obsolete) MMT 23400 # Myanmar Time - # (Asia/Rangoon) + # (Asia/Yangon) MYT 28800 # Malaysia Time # (Asia/Kuala_Lumpur) # (Asia/Kuching) @@ -183,12 +175,10 @@ NOVT Asia/Novosibirsk # Novosibirsk Time (obsolete) NPT 20700 # Nepal Time # (Asia/Katmandu) OMSST Asia/Omsk # Omsk Summer Time (obsolete) -OMST Asia/Omsk # Omsk Time - # (Asia/Omsk) +OMST Asia/Omsk # Omsk Time (obsolete) ORAT Asia/Oral # Oral Time (obsolete) PETST Asia/Kamchatka # Petropavlovsk-Kamchatski Summer Time (obsolete) -PETT Asia/Kamchatka # Petropavlovsk-Kamchatski Time - # (Asia/Kamchatka) +PETT Asia/Kamchatka # Petropavlovsk-Kamchatski Time (obsolete) PHT 28800 # Philippine Time # (Asia/Manila) PKT 18000 # Pakistan Time @@ -197,18 +187,14 @@ PKST 21600 D # Pakistan Summer Time # (Asia/Karachi) QYZT 21600 # Kizilorda Time (obsolete) SAKST Asia/Sakhalin # Sakhalin Summer Time (obsolete) -SAKT Asia/Sakhalin # Sakhalin Time - # (Asia/Sakhalin) +SAKT Asia/Sakhalin # Sakhalin Time (obsolete) SGT Asia/Singapore # Singapore Time # (Asia/Singapore) -SRET 39600 # Srednekolymsk Time - # (Asia/Srednekolymsk) -TJT 18000 # Tajikistan Time - # (Asia/Dushanbe) +SRET 39600 # Srednekolymsk Time (obsolete) +TJT 18000 # Tajikistan Time (obsolete) TLT 32400 # East Timor Time # (Asia/Dili) -TMT Asia/Ashgabat # Turkmenistan Time - # (Asia/Ashgabat) +TMT Asia/Ashgabat # Turkmenistan Time (obsolete) ULAST 32400 D # Ulan Bator Summer Time # (Asia/Ulaanbaatar) ULAT Asia/Ulaanbaatar # Ulan Bator Time @@ -216,12 +202,9 @@ ULAT Asia/Ulaanbaatar # Ulan Bator Time UZST 21600 D # Uzbekistan Summer Time # (Asia/Samarkand) # (Asia/Tashkent) -UZT 18000 # Uzbekistan Time - # (Asia/Samarkand) - # (Asia/Tashkent) +UZT 18000 # Uzbekistan Time (obsolete) VLAST Asia/Vladivostok # Vladivostok Summer Time (obsolete) -VLAT Asia/Vladivostok # Vladivostok Time - # (Asia/Vladivostok) +VLAT Asia/Vladivostok # Vladivostok Time (obsolete) WIB 25200 # Waktu Indonesia Barat # (Asia/Jakarta) # (Asia/Pontianak) @@ -232,8 +215,6 @@ WITA 28800 # Waktu Indonesia Tengah XJT 21600 # Xinjiang Time # (Asia/Urumqi) YAKST Asia/Yakutsk # Yakutsk Summer Time (obsolete) -YAKT Asia/Yakutsk # Yakutsk Time - # (Asia/Yakutsk) +YAKT Asia/Yakutsk # Yakutsk Time (obsolete) YEKST 21600 D # Yekaterinburg Summer Time (obsolete) -YEKT Asia/Yekaterinburg # Yekaterinburg Time - # (Asia/Yekaterinburg) +YEKT Asia/Yekaterinburg # Yekaterinburg Time (obsolete) diff --git a/src/timezone/tznames/Default b/src/timezone/tznames/Default index 7644e264b0..6f6e46e6b4 100644 --- a/src/timezone/tznames/Default +++ b/src/timezone/tznames/Default @@ -234,14 +234,9 @@ WGT -10800 # West Greenland Time #################### ANTARCTICA #################### -DAVT Antarctica/Davis # Davis Time (Antarctica) - # (Antarctica/Davis) -DDUT 36000 # Dumont-d'Urville Time (Antarctica) - # (Antarctica/DumontDUrville) - # (Antarctica/Palmer) - # (America/Santiago) -MAWT Antarctica/Mawson # Mawson Time (Antarctica) - # (Antarctica/Mawson) +DAVT Antarctica/Davis # Davis Time (Antarctica) (obsolete) +DDUT 36000 # Dumont-d'Urville Time (Antarctica) (obsolete) +MAWT Antarctica/Mawson # Mawson Time (Antarctica) (obsolete) #################### ASIA #################### @@ -256,15 +251,17 @@ AMST Asia/Yerevan # Armenia Summer Time # (Asia/Yerevan) # CONFLICT! AMT is not unique # Other timezones: -# - AMT: Amazon Time (America) -AMT Asia/Yerevan # Armenia Time - # (Asia/Yerevan) +# - AMT: Armenia Time (Asia) +AMT -14400 # Amazon Time + # (America/Boa_Vista) + # (America/Campo_Grande) + # (America/Cuiaba) + # (America/Manaus) + # (America/Porto_Velho) ANAST Asia/Anadyr # Anadyr Summer Time (obsolete) -ANAT Asia/Anadyr # Anadyr Time - # (Asia/Anadyr) +ANAT Asia/Anadyr # Anadyr Time (obsolete) AZST Asia/Baku # Azerbaijan Summer Time (obsolete) -AZT Asia/Baku # Azerbaijan Time - # (Asia/Baku) +AZT Asia/Baku # Azerbaijan Time (obsolete) BDT 21600 # Bangladesh Time # (Asia/Dhaka) BNT 28800 # Brunei Darussalam Time @@ -274,8 +271,7 @@ BTT 21600 # Bhutan Time # (Asia/Thimphu) CCT 28800 # China Coastal Time (not in IANA database) GEST Asia/Tbilisi # Georgia Summer Time (obsolete) -GET Asia/Tbilisi # Georgia Time - # (Asia/Tbilisi) +GET Asia/Tbilisi # Georgia Time (obsolete) HKT 28800 # Hong Kong Time (not in IANA database) ICT 25200 # Indochina Time # (Asia/Bangkok) @@ -285,8 +281,7 @@ ICT 25200 # Indochina Time IDT 10800 D # Israel Daylight Time # (Asia/Jerusalem) IRKST Asia/Irkutsk # Irkutsk Summer Time (obsolete) -IRKT Asia/Irkutsk # Irkutsk Time - # (Asia/Irkutsk) +IRKT Asia/Irkutsk # Irkutsk Time (obsolete) IRT 12600 # Iran Time (not in IANA database) # CONFLICT! IST is not unique # Other timezones: @@ -299,19 +294,16 @@ JST 32400 # Japan Standard Time # (Asia/Tokyo) KDT 36000 D # Korean Daylight Time (not in IANA database) KGST 21600 D # Kyrgyzstan Summer Time (obsolete) -KGT Asia/Bishkek # Kyrgyzstan Time - # (Asia/Bishkek) +KGT Asia/Bishkek # Kyrgyzstan Time (obsolete) KRAST Asia/Krasnoyarsk # Krasnoyarsk Summer Time (obsolete) -KRAT Asia/Krasnoyarsk # Krasnoyarsk Time - # (Asia/Krasnoyarsk) +KRAT Asia/Krasnoyarsk # Krasnoyarsk Time (obsolete) KST 32400 # Korean Standard Time # (Asia/Seoul) LKT Asia/Colombo # Lanka Time (obsolete) MAGST Asia/Magadan # Magadan Summer Time (obsolete) -MAGT Asia/Magadan # Magadan Time - # (Asia/Magadan) +MAGT Asia/Magadan # Magadan Time (obsolete) MMT 23400 # Myanmar Time - # (Asia/Rangoon) + # (Asia/Yangon) MYT 28800 # Malaysia Time # (Asia/Kuala_Lumpur) # (Asia/Kuching) @@ -320,11 +312,9 @@ NOVT Asia/Novosibirsk # Novosibirsk Time (obsolete) NPT 20700 # Nepal Time # (Asia/Katmandu) OMSST Asia/Omsk # Omsk Summer Time (obsolete) -OMST Asia/Omsk # Omsk Time - # (Asia/Omsk) +OMST Asia/Omsk # Omsk Time (obsolete) PETST Asia/Kamchatka # Petropavlovsk-Kamchatski Summer Time (obsolete) -PETT Asia/Kamchatka # Petropavlovsk-Kamchatski Time - # (Asia/Kamchatka) +PETT Asia/Kamchatka # Petropavlovsk-Kamchatski Time (obsolete) PHT 28800 # Philippine Time # (Asia/Manila) PKT 18000 # Pakistan Time @@ -333,10 +323,8 @@ PKST 21600 D # Pakistan Summer Time # (Asia/Karachi) SGT Asia/Singapore # Singapore Time # (Asia/Singapore) -TJT 18000 # Tajikistan Time - # (Asia/Dushanbe) -TMT Asia/Ashgabat # Turkmenistan Time - # (Asia/Ashgabat) +TJT 18000 # Tajikistan Time (obsolete) +TMT Asia/Ashgabat # Turkmenistan Time (obsolete) ULAST 32400 D # Ulan Bator Summer Time # (Asia/Ulaanbaatar) ULAT Asia/Ulaanbaatar # Ulan Bator Time @@ -344,20 +332,15 @@ ULAT Asia/Ulaanbaatar # Ulan Bator Time UZST 21600 D # Uzbekistan Summer Time # (Asia/Samarkand) # (Asia/Tashkent) -UZT 18000 # Uzbekistan Time - # (Asia/Samarkand) - # (Asia/Tashkent) +UZT 18000 # Uzbekistan Time (obsolete) VLAST Asia/Vladivostok # Vladivostok Summer Time (obsolete) -VLAT Asia/Vladivostok # Vladivostok Time - # (Asia/Vladivostok) +VLAT Asia/Vladivostok # Vladivostok Time (obsolete) XJT 21600 # Xinjiang Time # (Asia/Urumqi) YAKST Asia/Yakutsk # Yakutsk Summer Time (obsolete) -YAKT Asia/Yakutsk # Yakutsk Time - # (Asia/Yakutsk) +YAKT Asia/Yakutsk # Yakutsk Time (obsolete) YEKST 21600 D # Yekaterinburg Summer Time (obsolete) -YEKT Asia/Yekaterinburg # Yekaterinburg Time - # (Asia/Yekaterinburg) +YEKT Asia/Yekaterinburg # Yekaterinburg Time (obsolete) #################### ATLANTIC #################### @@ -669,8 +652,7 @@ RET 14400 # Reunion Time # (Indian/Reunion) SCT 14400 # Seychelles Time # (Indian/Mahe) -TFT 18000 # Kerguelen Time - # (Indian/Kerguelen) +TFT 18000 # Kerguelen Time (obsolete) #################### PACIFIC #################### diff --git a/src/timezone/tznames/Europe.txt b/src/timezone/tznames/Europe.txt index 5d6a594ab5..85d18d9d2d 100644 --- a/src/timezone/tznames/Europe.txt +++ b/src/timezone/tznames/Europe.txt @@ -191,8 +191,7 @@ MSK Europe/Moscow # Moscow Time # (Europe/Moscow) # (Europe/Volgograd) SAMST Europe/Samara # Samara Summer Time (obsolete) -SAMT Europe/Samara # Samara Time - # (Europe/Samara) +SAMT Europe/Samara # Samara Time (obsolete) VOLT Europe/Volgograd # Volgograd Time (obsolete) WEST 3600 D # Western Europe Summer Time # (Africa/Casablanca) diff --git a/src/timezone/tznames/Indian.txt b/src/timezone/tznames/Indian.txt index 634660075f..12842d366f 100644 --- a/src/timezone/tznames/Indian.txt +++ b/src/timezone/tznames/Indian.txt @@ -35,5 +35,4 @@ RET 14400 # Reunion Time # (Indian/Reunion) SCT 14400 # Seychelles Time # (Indian/Mahe) -TFT 18000 # Kerguelen Time - # (Indian/Kerguelen) +TFT 18000 # Kerguelen Time (obsolete) From a3215431ab7c667bf581728f10c80a36abbe1d5a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 19 Oct 2016 18:11:49 -0400 Subject: [PATCH 349/871] Suppress "Factory" zone in pg_timezone_names view for tzdata >= 2016g. IANA got rid of the really silly "abbreviation" and replaced it with one that's only moderately silly. But it's still pointless, so keep on not showing it. --- src/backend/utils/adt/datetime.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/datetime.c b/src/backend/utils/adt/datetime.c index 45ba7cd906..6d7fdc3c5e 100644 --- a/src/backend/utils/adt/datetime.c +++ b/src/backend/utils/adt/datetime.c @@ -4957,8 +4957,17 @@ pg_timezone_names(PG_FUNCTION_ARGS) &tzoff, &tm, &fsec, &tzn, tz) != 0) continue; /* ignore if conversion fails */ - /* Ignore zic's rather silly "Factory" time zone */ - if (tzn && strcmp(tzn, "Local time zone must be set--see zic manual page") == 0) + /* + * Ignore zic's rather silly "Factory" time zone. The long string + * about "see zic manual page" is used in tzdata versions before + * 2016g; we can drop it someday when we're pretty sure no such data + * exists in the wild on platforms using --with-system-tzdata. In + * 2016g and later, the time zone abbreviation "-00" is used for + * "Factory" as well as some invalid cases, all of which we can + * reasonably omit from the pg_timezone_names view. + */ + if (tzn && (strcmp(tzn, "-00") == 0 || + strcmp(tzn, "Local time zone must be set--see zic manual page") == 0)) continue; /* Found a displayable zone */ From f3094920a567cde6c86adf36a1a033d7431b11ff Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 19 Oct 2016 18:55:52 -0400 Subject: [PATCH 350/871] Sync our copy of the timezone library with IANA release tzcode2016g. This is mostly to absorb some corner-case fixes in zic for year-2037 timestamps. The other changes that have been made are unlikely to affect our usage, but nonetheless we may as well take 'em. --- src/timezone/localtime.c | 5 +- src/timezone/private.h | 1 + src/timezone/strftime.c | 2 +- src/timezone/zic.c | 440 ++++++++++++++++++++++----------------- 4 files changed, 250 insertions(+), 198 deletions(-) diff --git a/src/timezone/localtime.c b/src/timezone/localtime.c index d004e5ebe2..e6c1beaf96 100644 --- a/src/timezone/localtime.c +++ b/src/timezone/localtime.c @@ -1280,9 +1280,8 @@ gmtsub(pg_time_t const * timep, int32 offset, struct pg_tm * tmp) result = timesub(timep, offset, gmtptr, tmp); /* - * Could get fancy here and deliver something such as "UT+xxxx" or - * "UT-xxxx" if offset is non-zero, but this is no time for a treasure - * hunt. + * Could get fancy here and deliver something such as "+xx" or "-xx" if + * offset is non-zero, but this is no time for a treasure hunt. */ if (offset != 0) tmp->tm_zone = wildabbr; diff --git a/src/timezone/private.h b/src/timezone/private.h index 8480d38961..b8533d51e8 100644 --- a/src/timezone/private.h +++ b/src/timezone/private.h @@ -23,6 +23,7 @@ #include "pgtime.h" +/* This string was in the Factory zone through version 2016f. */ #define GRANDPARENTED "Local time zone must be set--see zic manual page" /* diff --git a/src/timezone/strftime.c b/src/timezone/strftime.c index 5630619321..4a0a01db65 100644 --- a/src/timezone/strftime.c +++ b/src/timezone/strftime.c @@ -128,7 +128,7 @@ pg_strftime(char *s, size_t maxsize, const char *format, int warn; warn = IN_NONE; - p = _fmt(((format == NULL) ? "%c" : format), t, s, s + maxsize, &warn); + p = _fmt(format, t, s, s + maxsize, &warn); if (p == s + maxsize) return 0; *p = '\0'; diff --git a/src/timezone/zic.c b/src/timezone/zic.c index b546a17372..52d8004ff7 100644 --- a/src/timezone/zic.c +++ b/src/timezone/zic.c @@ -105,7 +105,7 @@ static int addtype(zic_t, char const *, bool, bool, bool); static void leapadd(zic_t, bool, int, int); static void adjleap(void); static void associate(void); -static void dolink(const char *fromfield, const char *tofield); +static void dolink(const char *, const char *, bool); static char **getfields(char *buf); static zic_t gethms(const char *string, const char *errstring, bool); @@ -119,7 +119,7 @@ static bool inzsub(char **, int, bool); static int itsdir(const char *name); static bool is_alpha(char a); static char lowerit(char); -static bool mkdirs(char *); +static void mkdirs(char const *, bool); static void newabbr(const char *abbr); static zic_t oadd(zic_t t1, zic_t t2); static void outzone(const struct zone * zp, int ntzones); @@ -136,6 +136,15 @@ enum { PERCENT_Z_LEN_BOUND = sizeof "+995959" - 1}; +/* If true, work around a bug in Qt 5.6.1 and earlier, which mishandles + tz binary files whose POSIX-TZ-style strings contain '<'; see + QTBUG-53071 . This + workaround will no longer be needed when Qt 5.6.1 and earlier are + obsolete, say in the year 2021. */ +enum +{ +WORK_AROUND_QTBUG_53071 = true}; + static int charcnt; static bool errors; static bool warnings; @@ -346,6 +355,7 @@ static const int len_years[2] = { static struct attype { zic_t at; + bool dontmerge; unsigned char type; } *attypes; static zic_t gmtoffs[TZ_MAX_TYPES]; @@ -410,7 +420,8 @@ growalloc(void *ptr, size_t itemsize, int nitems, int *nitems_alloc) return ptr; else { - int amax = INT_MAX < SIZE_MAX ? INT_MAX : SIZE_MAX; + int nitems_max = INT_MAX - WORK_AROUND_QTBUG_53071; + int amax = nitems_max < SIZE_MAX ? nitems_max : SIZE_MAX; if ((amax - 1) / 3 * 2 < *nitems_alloc) memory_exhausted(_("int overflow")); @@ -478,17 +489,17 @@ warning(const char *string,...) } static void -close_file(FILE *stream, char const * name) +close_file(FILE *stream, char const * dir, char const * name) { char const *e = (ferror(stream) ? _("I/O error") : fclose(stream) != 0 ? strerror(errno) : NULL); if (e) { - fprintf(stderr, "%s: ", progname); - if (name) - fprintf(stderr, "%s: ", name); - fprintf(stderr, "%s\n", e); + fprintf(stderr, "%s: %s%s%s%s%s\n", progname, + dir ? dir : "", dir ? "/" : "", + name ? name : "", name ? ": " : "", + e); exit(EXIT_FAILURE); } } @@ -503,10 +514,34 @@ usage(FILE *stream, int status) "Report bugs to %s.\n"), progname, progname, PACKAGE_BUGREPORT); if (status == EXIT_SUCCESS) - close_file(stream, NULL); + close_file(stream, NULL, NULL); exit(status); } +/* Change the working directory to DIR, possibly creating DIR and its + ancestors. After this is done, all files are accessed with names + relative to DIR. */ +static void +change_directory(char const * dir) +{ + if (chdir(dir) != 0) + { + int chdir_errno = errno; + + if (chdir_errno == ENOENT) + { + mkdirs(dir, false); + chdir_errno = chdir(dir) == 0 ? 0 : errno; + } + if (chdir_errno != 0) + { + fprintf(stderr, _("%s: Can't chdir to %s: %s\n"), + progname, dir, strerror(chdir_errno)); + exit(EXIT_FAILURE); + } + } +} + static const char *psxrules; static const char *lcltime; static const char *directory; @@ -534,7 +569,7 @@ main(int argc, char *argv[]) if (strcmp(argv[i], "--version") == 0) { printf("zic %s\n", PG_VERSION); - close_file(stdout, NULL); + close_file(stdout, NULL, NULL); return EXIT_SUCCESS; } else if (strcmp(argv[i], "--help") == 0) @@ -630,6 +665,7 @@ main(int argc, char *argv[]) if (errors) return EXIT_FAILURE; associate(); + change_directory(directory); for (i = 0; i < nzones; i = j) { /* @@ -646,7 +682,7 @@ main(int argc, char *argv[]) for (i = 0; i < nlinks; ++i) { eat(links[i].l_filename, links[i].l_linenum); - dolink(links[i].l_from, links[i].l_to); + dolink(links[i].l_from, links[i].l_to, false); if (noise) for (j = 0; j < nlinks; ++j) if (strcmp(links[i].l_to, @@ -656,12 +692,12 @@ main(int argc, char *argv[]) if (lcltime != NULL) { eat(_("command line"), 1); - dolink(lcltime, TZDEFAULT); + dolink(lcltime, TZDEFAULT, true); } if (psxrules != NULL) { eat(_("command line"), 1); - dolink(psxrules, TZDEFRULES); + dolink(psxrules, TZDEFRULES, true); } if (warnings && (ferror(stderr) || fclose(stderr) != 0)) return EXIT_FAILURE; @@ -751,131 +787,117 @@ namecheck(const char *name) return componentcheck(name, component, cp); } -static char * -relname(char const * dir, char const * base) -{ - if (*base == '/') - return ecpyalloc(base); - else - { - size_t dir_len = strlen(dir); - bool needs_slash = dir_len && dir[dir_len - 1] != '/'; - char *result = emalloc(dir_len + needs_slash + strlen(base) + 1); - - result[dir_len] = '/'; - strcpy(result + dir_len + needs_slash, base); - return memcpy(result, dir, dir_len); - } -} - static void -dolink(char const * fromfield, char const * tofield) +dolink(char const * fromfield, char const * tofield, bool staysymlink) { - char *fromname; - char *toname; int fromisdir; - - fromname = relname(directory, fromfield); - toname = relname(directory, tofield); + bool todirs_made = false; + int link_errno; /* * We get to be careful here since there's a fair chance of root running * us. */ - fromisdir = itsdir(fromname); + fromisdir = itsdir(fromfield); if (fromisdir) { char const *e = strerror(fromisdir < 0 ? errno : EPERM); - fprintf(stderr, _("%s: link from %s failed: %s"), - progname, fromname, e); + fprintf(stderr, _("%s: link from %s/%s failed: %s\n"), + progname, directory, fromfield, e); exit(EXIT_FAILURE); } - if (link(fromname, toname) != 0) + if (staysymlink) + staysymlink = itsdir(tofield) == 2; + if (remove(tofield) == 0) + todirs_made = true; + else if (errno != ENOENT) { - int link_errno = errno; - bool retry_if_link_supported = false; + char const *e = strerror(errno); - if (link_errno == ENOENT || link_errno == ENOTSUP) + fprintf(stderr, _("%s: Can't remove %s/%s: %s\n"), + progname, directory, tofield, e); + exit(EXIT_FAILURE); + } + link_errno = (staysymlink ? ENOTSUP + : link(fromfield, tofield) == 0 ? 0 : errno); + if (link_errno == ENOENT && !todirs_made) + { + mkdirs(tofield, true); + todirs_made = true; + link_errno = link(fromfield, tofield) == 0 ? 0 : errno; + } + if (link_errno != 0) + { +#ifdef HAVE_SYMLINK + const char *s = fromfield; + const char *t; + char *p; + size_t dotdots = 0; + char *symlinkcontents; + int symlink_errno; + + do + t = s; + while ((s = strchr(s, '/')) + && strncmp(fromfield, tofield, ++s - fromfield) == 0); + + for (s = tofield + (t - fromfield); *s; s++) + dotdots += *s == '/'; + symlinkcontents = emalloc(3 * dotdots + strlen(t) + 1); + for (p = symlinkcontents; dotdots-- != 0; p += 3) + memcpy(p, "../", 3); + strcpy(p, t); + symlink_errno = symlink(symlinkcontents, tofield) == 0 ? 0 : errno; + if (symlink_errno == ENOENT && !todirs_made) { - if (!mkdirs(toname)) - exit(EXIT_FAILURE); - retry_if_link_supported = true; + mkdirs(tofield, true); + symlink_errno = symlink(symlinkcontents, tofield) == 0 ? 0 : errno; } - if ((link_errno == EEXIST || link_errno == ENOTSUP) - && itsdir(toname) == 0 - && (remove(toname) == 0 || errno == ENOENT)) - retry_if_link_supported = true; - if (retry_if_link_supported && link_errno != ENOTSUP) - link_errno = link(fromname, toname) == 0 ? 0 : errno; - if (link_errno != 0) + free(symlinkcontents); + if (symlink_errno == 0) { -#ifdef HAVE_SYMLINK - const char *s = fromfield; - const char *t; - char *p; - size_t dotdots = 0; - char *symlinkcontents; - int symlink_result; - - do - t = s; - while ((s = strchr(s, '/')) - && strncmp(fromfield, tofield, ++s - fromfield) == 0); - - for (s = tofield + (t - fromfield); *s; s++) - dotdots += *s == '/'; - symlinkcontents = emalloc(3 * dotdots + strlen(t) + 1); - for (p = symlinkcontents; dotdots-- != 0; p += 3) - memcpy(p, "../", 3); - strcpy(p, t); - symlink_result = symlink(symlinkcontents, toname); - free(symlinkcontents); - if (symlink_result == 0) - { - if (link_errno != ENOTSUP) - warning(_("symbolic link used because hard link failed: %s"), - strerror(link_errno)); - } - else + if (link_errno != ENOTSUP) + warning(_("symbolic link used because hard link failed: %s"), + strerror(link_errno)); + } + else #endif /* HAVE_SYMLINK */ - { - FILE *fp, - *tp; - int c; + { + FILE *fp, + *tp; + int c; - fp = fopen(fromname, "rb"); - if (!fp) - { - const char *e = strerror(errno); + fp = fopen(fromfield, "rb"); + if (!fp) + { + char const *e = strerror(errno); - fprintf(stderr, - _("%s: Can't read %s: %s\n"), - progname, fromname, e); - exit(EXIT_FAILURE); - } - tp = fopen(toname, "wb"); - if (!tp) - { - const char *e = strerror(errno); + fprintf(stderr, _("%s: Can't read %s/%s: %s\n"), + progname, directory, fromfield, e); + exit(EXIT_FAILURE); + } + tp = fopen(tofield, "wb"); + if (!tp) + { + char const *e = strerror(errno); - fprintf(stderr, - _("%s: Can't create %s: %s\n"), - progname, toname, e); - exit(EXIT_FAILURE); - } - while ((c = getc(fp)) != EOF) - putc(c, tp); - close_file(fp, fromname); - close_file(tp, toname); - if (link_errno != ENOTSUP) - warning(_("copy used because hard link failed: %s"), - strerror(link_errno)); + fprintf(stderr, _("%s: Can't create %s/%s: %s\n"), + progname, directory, tofield, e); + exit(EXIT_FAILURE); } + while ((c = getc(fp)) != EOF) + putc(c, tp); + close_file(fp, directory, fromfield); + close_file(tp, directory, tofield); + if (link_errno != ENOTSUP) + warning(_("copy used because hard link failed: %s"), + strerror(link_errno)); + else if (symlink_errno != ENOTSUP) + warning(_("copy used because symbolic link failed: %s"), + strerror(symlink_errno)); } } - free(fromname); - free(toname); } #define TIME_T_BITS_IN_FILE 64 @@ -888,10 +910,6 @@ static zic_t const max_time = MAXVAL(zic_t, TIME_T_BITS_IN_FILE); * rounded downward to the negation of a power of two that is * comfortably outside the error bounds. * - * zic does not output time stamps before this, partly because they - * are physically suspect, and partly because GNOME mishandles them; see - * GNOME bug 730332 . - * * For the time of the Big Bang, see: * * Ade PAR, Aghanim N, Armitage-Caplan C et al. Planck 2013 results. @@ -913,26 +931,45 @@ static zic_t const max_time = MAXVAL(zic_t, TIME_T_BITS_IN_FILE); #define BIG_BANG (- (((zic_t) 1) << 59)) #endif -static const zic_t big_bang_time = BIG_BANG; +/* If true, work around GNOME bug 730332 + + by refusing to output time stamps before BIG_BANG. + Such time stamps are physically suspect anyway. -/* Return 1 if NAME is a directory, 0 if it's something else, -1 if trouble. */ + The GNOME bug is scheduled to be fixed in GNOME 3.22, and if so + this workaround will no longer be needed when GNOME 3.21 and + earlier are obsolete, say in the year 2021. */ +enum +{ +WORK_AROUND_GNOME_BUG_730332 = true}; + +static const zic_t early_time = (WORK_AROUND_GNOME_BUG_730332 + ? BIG_BANG + : MINVAL(zic_t, TIME_T_BITS_IN_FILE)); + +/* Return 1 if NAME is a directory, 2 if a symbolic link, 0 if + something else, -1 (setting errno) if trouble. */ static int itsdir(char const * name) { struct stat st; - int res = stat(name, &st); + int res = lstat(name, &st); -#ifdef S_ISDIR if (res == 0) - return S_ISDIR(st.st_mode) != 0; -#endif - if (res == 0 || errno == EOVERFLOW) { - char *nameslashdot = relname(name, "."); - bool dir = stat(nameslashdot, &st) == 0 || errno == EOVERFLOW; +#ifdef S_ISDIR + return S_ISDIR(st.st_mode) ? 1 : S_ISLNK(st.st_mode) ? 2 : 0; +#else + size_t n = strlen(name); + char *nameslashdot = emalloc(n + 3); + bool dir; + memcpy(nameslashdot, name, n); + strcpy(&nameslashdot[n], &"/."[!(n && name[n - 1] != '/')]); + dir = lstat(nameslashdot, &st) == 0; free(nameslashdot); return dir; +#endif } return -1; } @@ -1129,7 +1166,7 @@ infile(const char *name) } free(fields); } - close_file(fp, filename); + close_file(fp, NULL, filename); if (wantcont) error(_("expected continuation line not found")); } @@ -1313,7 +1350,7 @@ inzsub(char **fields, int nfields, bool iscont) z.z_filename = filename; z.z_linenum = linenum; z.z_gmtoff = gethms(fields[i_gmtoff], _("invalid UT offset"), true); - if ((cp = strchr(fields[i_format], '%')) != 0) + if ((cp = strchr(fields[i_format], '%')) != NULL) { if ((*++cp != 's' && *cp != 'z') || strchr(cp, '%') || strchr(fields[i_format], '/')) @@ -1491,7 +1528,7 @@ inleap(char **fields, int nfields) return; } t = tadd(t, tod); - if (t < big_bang_time) + if (t < early_time) { error(_("leap second precedes Big Bang")); return; @@ -1764,11 +1801,14 @@ writezone(const char *const name, const char *const string, char version) int timecnt32, timei32; int pass; - char *fullname; static const struct tzhead tzh0; static struct tzhead tzh; - zic_t *ats = emalloc(size_product(timecnt, sizeof *ats + 1)); - void *typesptr = ats + timecnt; + bool dir_checked = false; + zic_t one = 1; + zic_t y2038_boundary = one << 31; + int nats = timecnt + WORK_AROUND_QTBUG_53071; + zic_t *ats = emalloc(size_product(nats, sizeof *ats + 1)); + void *typesptr = ats + nats; unsigned char *types = typesptr; /* @@ -1786,7 +1826,7 @@ writezone(const char *const name, const char *const string, char version) toi = 0; fromi = 0; - while (fromi < timecnt && attypes[fromi].at < big_bang_time) + while (fromi < timecnt && attypes[fromi].at < early_time) ++fromi; for (; fromi < timecnt; ++fromi) { @@ -1799,8 +1839,9 @@ writezone(const char *const name, const char *const string, char version) attypes[fromi].type; continue; } - if (toi == 0 || - attypes[toi - 1].type != attypes[fromi].type) + if (toi == 0 + || attypes[fromi].dontmerge + || attypes[toi - 1].type != attypes[fromi].type) attypes[toi++] = attypes[fromi]; } timecnt = toi; @@ -1818,6 +1859,20 @@ writezone(const char *const name, const char *const string, char version) types[i] = attypes[i].type; } + /* + * Work around QTBUG-53071 for time stamps less than y2038_boundary - 1, + * by inserting a no-op transition at time y2038_boundary - 1. This works + * only for timestamps before the boundary, which should be good enough in + * practice as QTBUG-53071 should be long-dead by 2038. + */ + if (WORK_AROUND_QTBUG_53071 && timecnt != 0 + && ats[timecnt - 1] < y2038_boundary - 1 && strchr(string, '<')) + { + ats[timecnt] = y2038_boundary - 1; + types[timecnt] = types[timecnt - 1]; + timecnt++; + } + /* * Correct for leap seconds. */ @@ -1862,29 +1917,35 @@ writezone(const char *const name, const char *const string, char version) --leapcnt32; ++leapi32; } - fullname = relname(directory, name); /* * Remove old file, if any, to snap links. */ - if (itsdir(fullname) == 0 && remove(fullname) != 0 && errno != ENOENT) + if (remove(name) == 0) + dir_checked = true; + else if (errno != ENOENT) { const char *e = strerror(errno); - fprintf(stderr, _("%s: Cannot remove %s: %s\n"), - progname, fullname, e); + fprintf(stderr, _("%s: Cannot remove %s/%s: %s\n"), + progname, directory, name, e); exit(EXIT_FAILURE); } - if ((fp = fopen(fullname, "wb")) == NULL) + fp = fopen(name, "wb"); + if (!fp) { - if (!mkdirs(fullname)) - exit(EXIT_FAILURE); - if ((fp = fopen(fullname, "wb")) == NULL) - { - const char *e = strerror(errno); + int fopen_errno = errno; - fprintf(stderr, _("%s: Cannot create %s: %s\n"), - progname, fullname, e); + if (fopen_errno == ENOENT && !dir_checked) + { + mkdirs(name, true); + fp = fopen(name, "wb"); + fopen_errno = errno; + } + if (!fp) + { + fprintf(stderr, _("%s: Cannot create %s/%s: %s\n"), + progname, directory, name, strerror(fopen_errno)); exit(EXIT_FAILURE); } } @@ -2130,9 +2191,8 @@ writezone(const char *const name, const char *const string, char version) putc(ttisgmts[i], fp); } fprintf(fp, "\n%s\n", string); - close_file(fp, fullname); + close_file(fp, directory, name); free(ats); - free(fullname); } static char const * @@ -2527,6 +2587,7 @@ outzone(const struct zone * zpfirst, int zonecount) int compat; bool do_extend; char version; + int lastatmax = -1; max_abbr_len = 2 + max_format_len + max_abbrvar_len; max_envvar_len = 2 * max_abbr_len + 5 * 9; @@ -2649,9 +2710,9 @@ outzone(const struct zone * zpfirst, int zonecount) */ stdoff = 0; zp = &zpfirst[i]; - usestart = i > 0 && (zp - 1)->z_untiltime > big_bang_time; + usestart = i > 0 && (zp - 1)->z_untiltime > early_time; useuntil = i < (zonecount - 1); - if (useuntil && zp->z_untiltime <= big_bang_time) + if (useuntil && zp->z_untiltime <= early_time) continue; gmtoff = zp->z_gmtoff; eat(zp->z_filename, zp->z_linenum); @@ -2670,7 +2731,7 @@ outzone(const struct zone * zpfirst, int zonecount) usestart = false; } else - addtt(big_bang_time, type); + addtt(early_time, type); } else for (year = min_year; year <= max_year; ++year) @@ -2792,6 +2853,10 @@ outzone(const struct zone * zpfirst, int zonecount) offset = oadd(zp->z_gmtoff, rp->r_stdoff); type = addtype(offset, ab, rp->r_stdoff != 0, rp->r_todisstd, rp->r_todisgmt); + if (rp->r_hiyear == ZIC_MAX + && !(0 <= lastatmax + && ktime < attypes[lastatmax].at)) + lastatmax = timecnt; addtt(ktime, type); } } @@ -2827,6 +2892,8 @@ outzone(const struct zone * zpfirst, int zonecount) starttime = tadd(starttime, -gmtoff); } } + if (0 <= lastatmax) + attypes[lastatmax].dontmerge = true; if (do_extend) { /* @@ -2850,22 +2917,8 @@ outzone(const struct zone * zpfirst, int zonecount) lastat = &attypes[i]; if (lastat->at < rpytime(&xr, max_year - 1)) { - /* - * Create new type code for the redundant entry, to prevent it - * being optimized away. - */ - if (typecnt >= TZ_MAX_TYPES) - { - error(_("too many local time types")); - exit(EXIT_FAILURE); - } - gmtoffs[typecnt] = gmtoffs[lastat->type]; - isdsts[typecnt] = isdsts[lastat->type]; - ttisstds[typecnt] = ttisstds[lastat->type]; - ttisgmts[typecnt] = ttisgmts[lastat->type]; - abbrinds[typecnt] = abbrinds[lastat->type]; - ++typecnt; addtt(rpytime(&xr, max_year + 1), typecnt - 1); + attypes[timecnt - 1].dontmerge = true; } } writezone(zpfirst->z_name, envvar, version); @@ -2877,8 +2930,8 @@ outzone(const struct zone * zpfirst, int zonecount) static void addtt(zic_t starttime, int type) { - if (starttime <= big_bang_time || - (timecnt == 1 && attypes[0].at < big_bang_time)) + if (starttime <= early_time + || (timecnt == 1 && attypes[0].at < early_time)) { gmtoffs[0] = gmtoffs[type]; isdsts[0] = isdsts[type]; @@ -2894,6 +2947,7 @@ addtt(zic_t starttime, int type) } attypes = growalloc(attypes, sizeof *attypes, timecnt, &timecnt_alloc); attypes[timecnt].at = starttime; + attypes[timecnt].dontmerge = false; attypes[timecnt].type = type; ++timecnt; } @@ -3441,53 +3495,51 @@ newabbr(const char *string) charcnt += i; } -static bool -mkdirs(char *argname) +/* Ensure that the directories of ARGNAME exist, by making any missing + ones. If ANCESTORS, do this only for ARGNAME's ancestors; otherwise, + do it for ARGNAME too. Exit with failure if there is trouble. */ +static void +mkdirs(char const * argname, bool ancestors) { char *name; char *cp; - if (argname == NULL || *argname == '\0') - return true; cp = name = ecpyalloc(argname); - while ((cp = strchr(cp + 1, '/')) != NULL) - { - *cp = '\0'; + + /* Do not mkdir a root directory, as it must exist. */ #ifdef WIN32 + if (is_alpha(name[0]) && name[1] == ':') + cp += 2; +#endif + while (*cp == '/') + cp++; - /* - * DOS drive specifier? - */ - if (is_alpha(name[0]) && name[1] == ':' && name[2] == '\0') - { - *cp = '/'; - continue; - } -#endif /* WIN32 */ + while (cp && ((cp = strchr(cp, '/')) || !ancestors)) + { + if (cp) + *cp = '\0'; /* * Try to create it. It's OK if creation fails because the directory * already exists, perhaps because some other process just created it. + * For simplicity do not check first whether it already exists, as + * that is checked anyway if the mkdir fails. */ if (mkdir(name, MKDIR_UMASK) != 0) { int err = errno; - if (itsdir(name) <= 0) + if (err != EEXIST && itsdir(name) < 0) { - char const *e = strerror(err); - - warning(_("%s: Can't create directory" - " %s: %s"), - progname, name, e); - free(name); - return false; + error(_("%s: Cannot create directory %s: %s"), + progname, name, strerror(err)); + exit(EXIT_FAILURE); } } - *cp = '/'; + if (cp) + *cp++ = '/'; } free(name); - return true; } From ad90ac4d671d320ade3c127f215e97cd49c307fb Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 19 Oct 2016 19:28:11 -0400 Subject: [PATCH 351/871] Windows portability fix. Per buildfarm. --- src/timezone/zic.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/timezone/zic.c b/src/timezone/zic.c index 52d8004ff7..04f4df27ce 100644 --- a/src/timezone/zic.c +++ b/src/timezone/zic.c @@ -35,6 +35,9 @@ typedef int64 zic_t; #define MKDIR_UMASK 0755 #endif #endif +#ifndef S_ISLNK +#define S_ISLNK(m) 0 +#endif struct rule { From 23ed2ba8121178474f8c51774c6c258cb165a562 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 19 Oct 2016 23:32:08 -0400 Subject: [PATCH 352/871] Another portability fix for tzcode2016g update. clang points out that SIZE_MAX wouldn't fit into an int, which means this comparison is pretty useless. Per report from Thomas Munro. --- src/timezone/zic.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/timezone/zic.c b/src/timezone/zic.c index 04f4df27ce..3f714ef46c 100644 --- a/src/timezone/zic.c +++ b/src/timezone/zic.c @@ -424,9 +424,8 @@ growalloc(void *ptr, size_t itemsize, int nitems, int *nitems_alloc) else { int nitems_max = INT_MAX - WORK_AROUND_QTBUG_53071; - int amax = nitems_max < SIZE_MAX ? nitems_max : SIZE_MAX; - if ((amax - 1) / 3 * 2 < *nitems_alloc) + if ((nitems_max - 1) / 3 * 2 < *nitems_alloc) memory_exhausted(_("int overflow")); *nitems_alloc = *nitems_alloc + (*nitems_alloc >> 1) + 1; return erealloc(ptr, size_product(*nitems_alloc, itemsize)); From ec7db2b483e0ff247ed41612cdb5716022401fe6 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 20 Oct 2016 10:23:39 -0400 Subject: [PATCH 353/871] Remove a comment which is now incorrect. Before 5d305d86bd917723f09ab4f15c075d90586a210a, this comment was correct, but now it says we do something which we don't actually do. Accordingly, remove the comment. --- src/interfaces/libpq/libpq-int.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index be6c3705b0..7007692fb1 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -364,7 +364,6 @@ struct pg_conn PGnotify *notifyTail; /* newest unreported Notify msg */ /* Connection data */ - /* See PQconnectPoll() for how we use 'int' and not 'pgsocket'. */ pgsocket sock; /* FD for socket, PGINVALID_SOCKET if * unconnected */ SockAddr laddr; /* Local address */ From f82ec32ac30ae7e3ec7c84067192535b2ff8ec0e Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 20 Oct 2016 11:24:37 -0400 Subject: [PATCH 354/871] Rename "pg_xlog" directory to "pg_wal". "xlog" is not a particularly clear abbreviation for "write-ahead log", and it sometimes confuses users into believe that the contents of the "pg_xlog" directory are not critical data, leading to unpleasant consequences. So, rename the directory to "pg_wal". This patch modifies pg_upgrade and pg_basebackup to understand both the old and new directory layouts; the former is necessary given the purpose of the tool, while the latter merely avoids an unnecessary backward-compatibility break. We may wish to consider renaming other programs, switches, and functions which still use the old "xlog" naming to also refer to "wal". However, that's still under discussion, so let's do just this much for now. Discussion: CAB7nPqTeC-8+zux8_-4ZD46V7YPwooeFxgndfsq5Rg8ibLVm1A@mail.gmail.com Michael Paquier --- doc/src/sgml/backup.sgml | 28 +++--- doc/src/sgml/config.sgml | 6 +- doc/src/sgml/func.sgml | 2 +- doc/src/sgml/high-availability.sgml | 14 +-- doc/src/sgml/perform.sgml | 2 +- doc/src/sgml/protocol.sgml | 6 +- doc/src/sgml/ref/pg_resetxlog.sgml | 10 +- doc/src/sgml/ref/pg_rewind.sgml | 4 +- doc/src/sgml/ref/pg_xlogdump.sgml | 2 +- doc/src/sgml/ref/pgtestfsync.sgml | 4 +- doc/src/sgml/ref/pgupgrade.sgml | 4 +- doc/src/sgml/storage.sgml | 2 +- doc/src/sgml/wal.sgml | 10 +- src/backend/access/transam/timeline.c | 4 +- src/backend/access/transam/xlog.c | 100 +++++++++---------- src/backend/access/transam/xlogarchive.c | 2 +- src/backend/access/transam/xlogfuncs.c | 2 +- src/backend/replication/README | 2 +- src/backend/replication/basebackup.c | 20 ++-- src/backend/replication/walreceiver.c | 6 +- src/backend/replication/walsender.c | 4 +- src/backend/storage/file/fd.c | 16 +-- src/bin/initdb/initdb.c | 12 +-- src/bin/pg_basebackup/pg_basebackup.c | 80 +++++++++------ src/bin/pg_basebackup/t/010_pg_basebackup.pl | 8 +- src/bin/pg_resetxlog/pg_resetxlog.c | 2 +- src/bin/pg_rewind/copy_fetch.c | 4 +- src/bin/pg_rewind/filemap.c | 8 +- src/bin/pg_rewind/parsexlog.c | 2 +- src/bin/pg_rewind/t/004_pg_xlog_symlink.pl | 10 +- src/bin/pg_upgrade/exec.c | 79 +++++++++------ src/bin/pg_xlogdump/pg_xlogdump.c | 2 +- src/common/file_utils.c | 39 +++++--- src/include/access/xlog_internal.h | 2 +- src/include/catalog/catversion.h | 2 +- src/include/common/file_utils.h | 3 +- src/include/postmaster/pgarch.h | 2 +- 37 files changed, 279 insertions(+), 226 deletions(-) diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 95d0ff3149..6eaed1efbe 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -472,7 +472,7 @@ tar -cf backup.tar /usr/local/pgsql/data At all times, PostgreSQL maintains a - write ahead log (WAL) in the pg_xlog/ + write ahead log (WAL) in the pg_wal/ subdirectory of the cluster's data directory. The log records every change made to the database's data files. This log exists primarily for crash-safety purposes: if the system crashes, the @@ -616,7 +616,7 @@ archive_command = 'copy "%p" "C:\\server\\archivedir\\%f"' # Windows %p and %f parameters have been replaced, the actual command executed might look like this: -test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_xlog/00000001000000A900000065 /mnt/server/archivedir/00000001000000A900000065 +test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_wal/00000001000000A900000065 /mnt/server/archivedir/00000001000000A900000065 A similar command will be generated for each new file to be archived. @@ -668,9 +668,9 @@ test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_xlog/ fills, nothing further can be archived until the tape is swapped. You should ensure that any error condition or request to a human operator is reported appropriately so that the situation can be - resolved reasonably quickly. The pg_xlog/ directory will + resolved reasonably quickly. The pg_wal/ directory will continue to fill with WAL segment files until the situation is resolved. - (If the file system containing pg_xlog/ fills up, + (If the file system containing pg_wal/ fills up, PostgreSQL will do a PANIC shutdown. No committed transactions will be lost, but the database will remain offline until you free some space.) @@ -682,7 +682,7 @@ test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_xlog/ operation continues even if the archiving process falls a little behind. If archiving falls significantly behind, this will increase the amount of data that would be lost in the event of a disaster. It will also mean that - the pg_xlog/ directory will contain large numbers of + the pg_wal/ directory will contain large numbers of not-yet-archived segment files, which could eventually exceed available disk space. You are advised to monitor the archiving process to ensure that it is working as you intend. @@ -743,7 +743,7 @@ test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_xlog/ configuration file reload. If you wish to temporarily stop archiving, one way to do it is to set archive_command to the empty string (''). - This will cause WAL files to accumulate in pg_xlog/ until a + This will cause WAL files to accumulate in pg_wal/ until a working archive_command is re-established. @@ -1062,10 +1062,10 @@ SELECT pg_stop_backup(); You should, however, omit from the backup the files within the - cluster's pg_xlog/ subdirectory. This + cluster's pg_wal/ subdirectory. This slight adjustment is worthwhile because it reduces the risk of mistakes when restoring. This is easy to arrange if - pg_xlog/ is a symbolic link pointing to someplace outside + pg_wal/ is a symbolic link pointing to someplace outside the cluster directory, which is a common setup anyway for performance reasons. You might also want to exclude postmaster.pid and postmaster.opts, which record information @@ -1149,7 +1149,7 @@ SELECT pg_stop_backup(); location in case you need them later. Note that this precaution will require that you have enough free space on your system to hold two copies of your existing database. If you do not have enough space, - you should at least save the contents of the cluster's pg_xlog + you should at least save the contents of the cluster's pg_wal subdirectory, as it might contain logs which were not archived before the system went down. @@ -1172,9 +1172,9 @@ SELECT pg_stop_backup(); - Remove any files present in pg_xlog/; these came from the + Remove any files present in pg_wal/; these came from the file system backup and are therefore probably obsolete rather than current. - If you didn't archive pg_xlog/ at all, then recreate + If you didn't archive pg_wal/ at all, then recreate it with proper permissions, being careful to ensure that you re-establish it as a symbolic link if you had it set up that way before. @@ -1183,7 +1183,7 @@ SELECT pg_stop_backup(); If you have unarchived WAL segment files that you saved in step 2, - copy them into pg_xlog/. (It is best to copy them, + copy them into pg_wal/. (It is best to copy them, not move them, so you still have the unmodified files if a problem occurs and you have to start over.) @@ -1265,9 +1265,9 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' WAL segments that cannot be found in the archive will be sought in - pg_xlog/; this allows use of recent un-archived segments. + pg_wal/; this allows use of recent un-archived segments. However, segments that are available from the archive will be used in - preference to files in pg_xlog/. + preference to files in pg_wal/. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 99ff9f5ab5..adab2f8378 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2932,7 +2932,7 @@ include_dir 'conf.d' Specifies the minimum number of past log file segments kept in the - pg_xlog + pg_wal directory, in case a standby server needs to fetch them for streaming replication. Each segment is normally 16 megabytes. If a standby server connected to the sending server falls behind by more than @@ -2946,7 +2946,7 @@ include_dir 'conf.d' This sets only the minimum number of segments retained in - pg_xlog; the system might need to retain more segments + pg_wal; the system might need to retain more segments for WAL archival or to recover from a checkpoint. If wal_keep_segments is zero (the default), the system doesn't keep any extra segments for standby purposes, so the number @@ -3322,7 +3322,7 @@ include_dir 'conf.d' Specify how long the standby server should wait when WAL data is not available from any sources (streaming replication, - local pg_xlog or WAL archive) before retrying to + local pg_wal or WAL archive) before retrying to retrieve WAL data. This parameter can only be set in the postgresql.conf file or on the server command line. The default value is 5 seconds. Units are milliseconds if not specified. diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index a58835082b..2e64cc430c 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -15327,7 +15327,7 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); pg_snapshots | 13 pg_multixact | 14 PG_VERSION | 15 - pg_xlog | 16 + pg_wal | 16 pg_hba.conf | 17 pg_stat_tmp | 18 pg_subtrans | 19 diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 06f49dba5d..5bedaf27a2 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -594,24 +594,24 @@ protocol to make nodes agree on a serializable transactional order. (see ) or directly from the master over a TCP connection (streaming replication). The standby server will also attempt to restore any WAL found in the standby cluster's - pg_xlog directory. That typically happens after a server + pg_wal directory. That typically happens after a server restart, when the standby replays again WAL that was streamed from the master before the restart, but you can also manually copy files to - pg_xlog at any time to have them replayed. + pg_wal at any time to have them replayed. At startup, the standby begins by restoring all WAL available in the archive location, calling restore_command. Once it reaches the end of WAL available there and restore_command - fails, it tries to restore any WAL available in the pg_xlog directory. + fails, it tries to restore any WAL available in the pg_wal directory. If that fails, and streaming replication has been configured, the standby tries to connect to the primary server and start streaming WAL - from the last valid record found in archive or pg_xlog. If that fails + from the last valid record found in archive or pg_wal. If that fails or streaming replication is not configured, or if the connection is later disconnected, the standby goes back to step 1 and tries to restore the file from the archive again. This loop of retries from the - archive, pg_xlog, and via streaming replication goes on until the server + archive, pg_wal, and via streaming replication goes on until the server is stopped or failover is triggered by a trigger file. @@ -619,7 +619,7 @@ protocol to make nodes agree on a serializable transactional order. Standby mode is exited and the server switches to normal operation when pg_ctl promote is run or a trigger file is found (trigger_file). Before failover, - any WAL immediately available in the archive or in pg_xlog will be + any WAL immediately available in the archive or in pg_wal will be restored, but no attempt is made to connect to the master. @@ -895,7 +895,7 @@ primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass' However, these methods often result in retaining more WAL segments than required, whereas replication slots retain only the number of segments known to be needed. An advantage of these methods is that they bound - the space requirement for pg_xlog; there is currently no way + the space requirement for pg_wal; there is currently no way to do this using replication slots. diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index 7bcbfa7611..8d30fd1384 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1612,7 +1612,7 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; Increase and ; this reduces the frequency of checkpoints, but increases the storage requirements of - /pg_xlog. + /pg_wal. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 3384e73448..50cf527427 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1947,7 +1947,7 @@ The commands accepted in walsender mode are: Include the necessary WAL segments in the backup. This will include all the files between start and stop backup in the - pg_xlog directory of the base directory tar + pg_wal directory of the base directory tar file. @@ -2076,8 +2076,8 @@ The commands accepted in walsender mode are: - pg_xlog, including subdirectories. If the backup is run - with WAL files included, a synthesized version of pg_xlog will be + pg_wal, including subdirectories. If the backup is run + with WAL files included, a synthesized version of pg_wal will be included, but it will only contain the files necessary for the backup to work, not the rest of the contents. diff --git a/doc/src/sgml/ref/pg_resetxlog.sgml b/doc/src/sgml/ref/pg_resetxlog.sgml index fd9d0be6f4..c949c5e849 100644 --- a/doc/src/sgml/ref/pg_resetxlog.sgml +++ b/doc/src/sgml/ref/pg_resetxlog.sgml @@ -173,22 +173,22 @@ PostgreSQL documentation The WAL starting address should be larger than any WAL segment file name currently existing in - the directory pg_xlog under the data directory. + the directory pg_wal under the data directory. These names are also in hexadecimal and have three parts. The first part is the timeline ID and should usually be kept the same. For example, if 00000001000000320000004A is the - largest entry in pg_xlog, use -l 00000001000000320000004B or higher. + largest entry in pg_wal, use -l 00000001000000320000004B or higher. pg_resetxlog itself looks at the files in - pg_xlog and chooses a default diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index 42ebfbfdef..371c4a475f 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -61,14 +61,14 @@ PostgreSQL documentation pg_rewind examines the timeline histories of the source and target clusters to determine the point where they diverged, and - expects to find WAL in the target cluster's pg_xlog directory + expects to find WAL in the target cluster's pg_wal directory reaching all the way back to the point of divergence. The point of divergence can be found either on the target timeline, the source timeline, or their common ancestor. In the typical failover scenario where the target cluster was shut down soon after the divergence, this is not a problem, but if the target cluster ran for a long time after the divergence, the old WAL files might no longer be present. In that case, they can be manually - copied from the WAL archive to the pg_xlog directory, or + copied from the WAL archive to the pg_wal directory, or fetched on startup by configuring recovery.conf. The use of pg_rewind is not limited to failover, e.g. a standby server can be promoted, run some write transactions, and then rewinded diff --git a/doc/src/sgml/ref/pg_xlogdump.sgml b/doc/src/sgml/ref/pg_xlogdump.sgml index 177caab00d..cfb6d87259 100644 --- a/doc/src/sgml/ref/pg_xlogdump.sgml +++ b/doc/src/sgml/ref/pg_xlogdump.sgml @@ -118,7 +118,7 @@ PostgreSQL documentation Directory in which to find log segment files. The default is to search - for them in the pg_xlog subdirectory of the current + for them in the pg_wal subdirectory of the current directory. diff --git a/doc/src/sgml/ref/pgtestfsync.sgml b/doc/src/sgml/ref/pgtestfsync.sgml index 6e134c75df..5856356b42 100644 --- a/doc/src/sgml/ref/pgtestfsync.sgml +++ b/doc/src/sgml/ref/pgtestfsync.sgml @@ -57,8 +57,8 @@ Specifies the file name to write test data in. This file should be in the same file system that the - pg_xlog directory is or will be placed in. - (pg_xlog contains the WAL files.) + pg_wal directory is or will be placed in. + (pg_wal contains the WAL files.) The default is pg_test_fsync.out in the current directory. diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index d46a730f66..ad28526296 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -345,7 +345,7 @@ NET STOP postgresql-9.0 your old cluster once you start the new cluster after the upgrade. Link mode also requires that the old and new cluster data directories be in the - same file system. (Tablespaces and pg_xlog can be on + same file system. (Tablespaces and pg_wal can be on different file systems.) See pg_upgrade --help for a full list of options. @@ -508,7 +508,7 @@ rsync --archive --delete --hard-links --size-only old_pgdata new_pgdata remote_d If you have tablespaces, you will need to run a similar rsync command for each tablespace directory. If you - have relocated pg_xlog outside the data directories, + have relocated pg_wal outside the data directories, rsync must be run on those directories too. diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 1b812bd0a9..fddb69bad3 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -141,7 +141,7 @@ Item - pg_xlog + pg_wal Subdirectory containing WAL (Write Ahead Log) files diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index fe3b588c72..346aa769a8 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -557,7 +557,7 @@ - The number of WAL segment files in pg_xlog directory depends on + The number of WAL segment files in pg_wal directory depends on min_wal_size, max_wal_size and the amount of WAL generated in previous checkpoint cycles. When old log segment files are no longer needed, they are removed or recycled (that is, @@ -582,7 +582,7 @@ kept at all times. Also, if WAL archiving is used, old segments can not be removed or recycled until they are archived. If WAL archiving cannot keep up with the pace that WAL is generated, or if archive_command - fails repeatedly, old WAL files will accumulate in pg_xlog + fails repeatedly, old WAL files will accumulate in pg_wal until the situation is resolved. A slow or failed standby server that uses a replication slot will have the same effect (see ). @@ -594,7 +594,7 @@ which are similar to checkpoints in normal operation: the server forces all its state to disk, updates the pg_control file to indicate that the already-processed WAL data need not be scanned again, - and then recycles any old log segment files in the pg_xlog + and then recycles any old log segment files in the pg_wal directory. Restartpoints can't be performed more frequently than checkpoints in the master because restartpoints can only be performed at checkpoint records. @@ -750,7 +750,7 @@ WAL logs are stored in the directory - pg_xlog under the data directory, as a set of + pg_wal under the data directory, as a set of segment files, normally each 16 MB in size (but the size can be changed by altering the + + The transaction log files are written to a separate file + named pg_wal.tar (if the server is a version + earlier than 10, the file will be named pg_xlog.tar). + @@ -353,7 +363,8 @@ PostgreSQL documentation Enables gzip compression of tar file output, with the default compression level. Compression is only available when using - the tar format. + the tar format, and the suffix .gz will + automatically be added to all tar filenames. @@ -366,7 +377,8 @@ PostgreSQL documentation Enables gzip compression of tar file output, and specifies the compression level (0 through 9, 0 being no compression and 9 being best compression). Compression is only available when using the tar - format. + format, and the suffix .gz will + automatically be added to all tar filenames. diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile index fa1ce8b24d..52ac9e9fb8 100644 --- a/src/bin/pg_basebackup/Makefile +++ b/src/bin/pg_basebackup/Makefile @@ -19,7 +19,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpq -OBJS=receivelog.o streamutil.o $(WIN32RES) +OBJS=receivelog.o streamutil.o walmethods.o $(WIN32RES) all: pg_basebackup pg_receivexlog pg_recvlogical diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index b82b8e1b26..16cab978d0 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -449,7 +449,7 @@ typedef struct { PGconn *bgconn; XLogRecPtr startptr; - char xlogdir[MAXPGPATH]; + char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */ char *sysidentifier; int timeline; } logstreamer_param; @@ -470,9 +470,13 @@ LogStreamerMain(logstreamer_param *param) stream.synchronous = false; stream.do_sync = do_sync; stream.mark_done = true; - stream.basedir = param->xlogdir; stream.partial_suffix = NULL; + if (format == 'p') + stream.walmethod = CreateWalDirectoryMethod(param->xlog, do_sync); + else + stream.walmethod = CreateWalTarMethod(param->xlog, compresslevel, do_sync); + if (!ReceiveXlogStream(param->bgconn, &stream)) /* @@ -482,6 +486,14 @@ LogStreamerMain(logstreamer_param *param) */ return 1; + if (!stream.walmethod->finish()) + { + fprintf(stderr, + _("%s: could not finish writing WAL files: %s\n"), + progname, strerror(errno)); + return 1; + } + PQfinish(param->bgconn); return 0; } @@ -533,28 +545,32 @@ StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier) exit(1); /* In post-10 cluster, pg_xlog has been renamed to pg_wal */ - snprintf(param->xlogdir, sizeof(param->xlogdir), "%s/%s", + snprintf(param->xlog, sizeof(param->xlog), "%s/%s", basedir, PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ? "pg_xlog" : "pg_wal"); - /* - * Create pg_wal/archive_status or pg_xlog/archive_status (and thus - * pg_wal or pg_xlog) depending on the target server so we can write to - * basedir/pg_wal or basedir/pg_xlog as the directory entry in the tar - * file may arrive later. - */ - snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status", - basedir, - PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ? - "pg_xlog" : "pg_wal"); - if (pg_mkdir_p(statusdir, S_IRWXU) != 0 && errno != EEXIST) + if (format == 'p') { - fprintf(stderr, - _("%s: could not create directory \"%s\": %s\n"), - progname, statusdir, strerror(errno)); - disconnect_and_exit(1); + /* + * Create pg_wal/archive_status or pg_xlog/archive_status (and thus + * pg_wal or pg_xlog) depending on the target server so we can write to + * basedir/pg_wal or basedir/pg_xlog as the directory entry in the tar + * file may arrive later. + */ + snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status", + basedir, + PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ? + "pg_xlog" : "pg_wal"); + + if (pg_mkdir_p(statusdir, S_IRWXU) != 0 && errno != EEXIST) + { + fprintf(stderr, + _("%s: could not create directory \"%s\": %s\n"), + progname, statusdir, strerror(errno)); + disconnect_and_exit(1); + } } /* @@ -2245,16 +2261,6 @@ main(int argc, char **argv) exit(1); } - if (format != 'p' && streamwal) - { - fprintf(stderr, - _("%s: WAL streaming can only be used in plain mode\n"), - progname); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), - progname); - exit(1); - } - if (replication_slot && !streamwal) { fprintf(stderr, diff --git a/src/bin/pg_basebackup/pg_receivexlog.c b/src/bin/pg_basebackup/pg_receivexlog.c index a58a251a59..bbdf96edfd 100644 --- a/src/bin/pg_basebackup/pg_receivexlog.c +++ b/src/bin/pg_basebackup/pg_receivexlog.c @@ -338,11 +338,19 @@ StreamLog(void) stream.synchronous = synchronous; stream.do_sync = true; stream.mark_done = false; - stream.basedir = basedir; + stream.walmethod = CreateWalDirectoryMethod(basedir, stream.do_sync); stream.partial_suffix = ".partial"; ReceiveXlogStream(conn, &stream); + if (!stream.walmethod->finish()) + { + fprintf(stderr, + _("%s: could not finish writing WAL files: %s\n"), + progname, strerror(errno)); + return; + } + PQfinish(conn); conn = NULL; } diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index b0fa916b44..fcd0269473 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -30,7 +30,7 @@ /* fd and filename for currently open WAL file */ -static int walfile = -1; +static Walfile *walfile = NULL; static char current_walfile_name[MAXPGPATH] = ""; static bool reportFlushPosition = false; static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr; @@ -56,29 +56,23 @@ static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos, uint32 *timeline); static bool -mark_file_as_archived(const char *basedir, const char *fname, bool do_sync) +mark_file_as_archived(StreamCtl *stream, const char *fname) { - int fd; + Walfile *f; static char tmppath[MAXPGPATH]; - snprintf(tmppath, sizeof(tmppath), "%s/archive_status/%s.done", - basedir, fname); + snprintf(tmppath, sizeof(tmppath), "archive_status/%s.done", + fname); - fd = open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); - if (fd < 0) + f = stream->walmethod->open_for_write(tmppath, NULL, 0); + if (f == NULL) { fprintf(stderr, _("%s: could not create archive status file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); + progname, tmppath, stream->walmethod->getlasterror()); return false; } - close(fd); - - if (do_sync && fsync_fname(tmppath, false, progname) != 0) - return false; - - if (do_sync && fsync_parent_path(tmppath, progname) != 0) - return false; + stream->walmethod->close(f, CLOSE_NORMAL); return true; } @@ -95,121 +89,82 @@ mark_file_as_archived(const char *basedir, const char *fname, bool do_sync) static bool open_walfile(StreamCtl *stream, XLogRecPtr startpoint) { - int f; + Walfile *f; char fn[MAXPGPATH]; - struct stat statbuf; - char *zerobuf; - int bytes; + ssize_t size; XLogSegNo segno; XLByteToSeg(startpoint, segno); XLogFileName(current_walfile_name, stream->timeline, segno); - snprintf(fn, sizeof(fn), "%s/%s%s", stream->basedir, current_walfile_name, + snprintf(fn, sizeof(fn), "%s%s", current_walfile_name, stream->partial_suffix ? stream->partial_suffix : ""); - f = open(fn, O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); - if (f == -1) - { - fprintf(stderr, - _("%s: could not open transaction log file \"%s\": %s\n"), - progname, fn, strerror(errno)); - return false; - } /* - * Verify that the file is either empty (just created), or a complete - * XLogSegSize segment. Anything in between indicates a corrupt file. + * When streaming to files, if an existing file exists we verify that it's + * either empty (just created), or a complete XLogSegSize segment (in + * which case it has been created and padded). Anything else indicates a + * corrupt file. + * + * When streaming to tar, no file with this name will exist before, so we + * never have to verify a size. */ - if (fstat(f, &statbuf) != 0) + if (stream->walmethod->existsfile(fn)) { - fprintf(stderr, - _("%s: could not stat transaction log file \"%s\": %s\n"), - progname, fn, strerror(errno)); - close(f); - return false; - } - if (statbuf.st_size == XLogSegSize) - { - /* - * fsync, in case of a previous crash between padding and fsyncing the - * file. - */ - if (stream->do_sync) + size = stream->walmethod->get_file_size(fn); + if (size < 0) { - if (fsync_fname(fn, false, progname) != 0 || - fsync_parent_path(fn, progname) != 0) + fprintf(stderr, + _("%s: could not get size of transaction log file \"%s\": %s\n"), + progname, fn, stream->walmethod->getlasterror()); + return false; + } + if (size == XLogSegSize) + { + /* Already padded file. Open it for use */ + f = stream->walmethod->open_for_write(current_walfile_name, stream->partial_suffix, 0); + if (f == NULL) { - /* error already printed */ - close(f); + fprintf(stderr, + _("%s: could not open existing transaction log file \"%s\": %s\n"), + progname, fn, stream->walmethod->getlasterror()); return false; } - } - /* File is open and ready to use */ - walfile = f; - return true; - } - if (statbuf.st_size != 0) - { - fprintf(stderr, - _("%s: transaction log file \"%s\" has %d bytes, should be 0 or %d\n"), - progname, fn, (int) statbuf.st_size, XLogSegSize); - close(f); - return false; - } + /* fsync file in case of a previous crash */ + if (!stream->walmethod->fsync(f)) + { + stream->walmethod->close(f, CLOSE_UNLINK); + return false; + } - /* - * New, empty, file. So pad it to 16Mb with zeroes. If we fail partway - * through padding, we should attempt to unlink the file on failure, so as - * not to leave behind a partially-filled file. - */ - zerobuf = pg_malloc0(XLOG_BLCKSZ); - for (bytes = 0; bytes < XLogSegSize; bytes += XLOG_BLCKSZ) - { - errno = 0; - if (write(f, zerobuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + walfile = f; + return true; + } + if (size != 0) { /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) errno = ENOSPC; fprintf(stderr, - _("%s: could not pad transaction log file \"%s\": %s\n"), - progname, fn, strerror(errno)); - free(zerobuf); - close(f); - unlink(fn); + _("%s: transaction log file \"%s\" has %d bytes, should be 0 or %d\n"), + progname, fn, (int) size, XLogSegSize); return false; } + /* File existed and was empty, so fall through and open */ } - free(zerobuf); - /* - * fsync WAL file and containing directory, to ensure the file is - * persistently created and zeroed. That's particularly important when - * using synchronous mode, where the file is modified and fsynced - * in-place, without a directory fsync. - */ - if (stream->do_sync) - { - if (fsync_fname(fn, false, progname) != 0 || - fsync_parent_path(fn, progname) != 0) - { - /* error already printed */ - close(f); - return false; - } - } + /* No file existed, so create one */ - if (lseek(f, SEEK_SET, 0) != 0) + f = stream->walmethod->open_for_write(current_walfile_name, stream->partial_suffix, XLogSegSize); + if (f == NULL) { fprintf(stderr, - _("%s: could not seek to beginning of transaction log file \"%s\": %s\n"), - progname, fn, strerror(errno)); - close(f); + _("%s: could not open transaction log file \"%s\": %s\n"), + progname, fn, stream->walmethod->getlasterror()); return false; } - /* File is open and ready to use */ walfile = f; return true; } @@ -223,59 +178,46 @@ static bool close_walfile(StreamCtl *stream, XLogRecPtr pos) { off_t currpos; + int r; - if (walfile == -1) + if (walfile == NULL) return true; - currpos = lseek(walfile, 0, SEEK_CUR); + currpos = stream->walmethod->get_current_pos(walfile); if (currpos == -1) { fprintf(stderr, _("%s: could not determine seek position in file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); - close(walfile); - walfile = -1; + progname, current_walfile_name, stream->walmethod->getlasterror()); + stream->walmethod->close(walfile, CLOSE_UNLINK); + walfile = NULL; + return false; } - if (stream->do_sync && fsync(walfile) != 0) + if (stream->partial_suffix) { - fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); - close(walfile); - walfile = -1; - return false; + if (currpos == XLOG_SEG_SIZE) + r = stream->walmethod->close(walfile, CLOSE_NORMAL); + else + { + fprintf(stderr, + _("%s: not renaming \"%s%s\", segment is not complete\n"), + progname, current_walfile_name, stream->partial_suffix); + r = stream->walmethod->close(walfile, CLOSE_NO_RENAME); + } } + else + r = stream->walmethod->close(walfile, CLOSE_NORMAL); - if (close(walfile) != 0) + walfile = NULL; + + if (r != 0) { fprintf(stderr, _("%s: could not close file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); - walfile = -1; + progname, current_walfile_name, stream->walmethod->getlasterror()); return false; } - walfile = -1; - - /* - * If we finished writing a .partial file, rename it into place. - */ - if (currpos == XLOG_SEG_SIZE && stream->partial_suffix) - { - char oldfn[MAXPGPATH]; - char newfn[MAXPGPATH]; - - snprintf(oldfn, sizeof(oldfn), "%s/%s%s", stream->basedir, current_walfile_name, stream->partial_suffix); - snprintf(newfn, sizeof(newfn), "%s/%s", stream->basedir, current_walfile_name); - if (durable_rename(oldfn, newfn, progname) != 0) - { - /* durable_rename produced a log entry */ - return false; - } - } - else if (stream->partial_suffix) - fprintf(stderr, - _("%s: not renaming \"%s%s\", segment is not complete\n"), - progname, current_walfile_name, stream->partial_suffix); /* * Mark file as archived if requested by the caller - pg_basebackup needs @@ -286,8 +228,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) if (currpos == XLOG_SEG_SIZE && stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, current_walfile_name, - stream->do_sync)) + if (!mark_file_as_archived(stream, current_walfile_name)) return false; } @@ -302,9 +243,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) static bool existsTimeLineHistoryFile(StreamCtl *stream) { - char path[MAXPGPATH]; char histfname[MAXFNAMELEN]; - int fd; /* * Timeline 1 never has a history file. We treat that as if it existed, @@ -315,31 +254,15 @@ existsTimeLineHistoryFile(StreamCtl *stream) TLHistoryFileName(histfname, stream->timeline); - snprintf(path, sizeof(path), "%s/%s", stream->basedir, histfname); - - fd = open(path, O_RDONLY | PG_BINARY, 0); - if (fd < 0) - { - if (errno != ENOENT) - fprintf(stderr, _("%s: could not open timeline history file \"%s\": %s\n"), - progname, path, strerror(errno)); - return false; - } - else - { - close(fd); - return true; - } + return stream->walmethod->existsfile(histfname); } static bool writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) { int size = strlen(content); - char path[MAXPGPATH]; - char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; - int fd; + Walfile *f; /* * Check that the server's idea of how timeline history files should be @@ -353,53 +276,31 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) return false; } - snprintf(path, sizeof(path), "%s/%s", stream->basedir, histfname); - - /* - * Write into a temp file name. - */ - snprintf(tmppath, MAXPGPATH, "%s.tmp", path); - - unlink(tmppath); - - fd = open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); - if (fd < 0) + f = stream->walmethod->open_for_write(histfname, ".tmp", 0); + if (f == NULL) { fprintf(stderr, _("%s: could not create timeline history file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); + progname, histfname, stream->walmethod->getlasterror()); return false; } - errno = 0; - if ((int) write(fd, content, size) != size) + if ((int) stream->walmethod->write(f, content, size) != size) { - int save_errno = errno; + fprintf(stderr, _("%s: could not write timeline history file \"%s\": %s\n"), + progname, histfname, stream->walmethod->getlasterror()); /* * If we fail to make the file, delete it to release disk space */ - close(fd); - unlink(tmppath); - errno = save_errno; + stream->walmethod->close(f, CLOSE_UNLINK); - fprintf(stderr, _("%s: could not write timeline history file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); return false; } - if (close(fd) != 0) + if (stream->walmethod->close(f, CLOSE_NORMAL) != 0) { fprintf(stderr, _("%s: could not close file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); - return false; - } - - /* - * Now move the completed history file into place with its final name. - */ - if (durable_rename(tmppath, path, progname) < 0) - { - /* durable_rename produced a log entry */ + progname, histfname, stream->walmethod->getlasterror()); return false; } @@ -407,8 +308,7 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) if (stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, histfname, - stream->do_sync)) + if (!mark_file_as_archived(stream, histfname)) return false; } @@ -618,7 +518,9 @@ ReceiveXlogStream(PGconn *conn, StreamCtl *stream) { /* * Fetch the timeline history file for this timeline, if we don't have - * it already. + * it already. When streaming log to tar, this will always return + * false, as we are never streaming into an existing file and + * therefore there can be no pre-existing timeline history file. */ if (!existsTimeLineHistoryFile(stream)) { @@ -777,10 +679,10 @@ ReceiveXlogStream(PGconn *conn, StreamCtl *stream) } error: - if (walfile != -1 && close(walfile) != 0) + if (walfile != NULL && stream->walmethod->close(walfile, CLOSE_NORMAL) != 0) fprintf(stderr, _("%s: could not close file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); - walfile = -1; + progname, current_walfile_name, stream->walmethod->getlasterror()); + walfile = NULL; return false; } @@ -864,12 +766,12 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, * If synchronous option is true, issue sync command as soon as there * are WAL data which has not been flushed yet. */ - if (stream->synchronous && lastFlushPosition < blockpos && walfile != -1) + if (stream->synchronous && lastFlushPosition < blockpos && walfile != NULL) { - if (stream->do_sync && fsync(walfile) != 0) + if (stream->walmethod->fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); + progname, current_walfile_name, stream->walmethod->getlasterror()); goto error; } lastFlushPosition = blockpos; @@ -1100,7 +1002,7 @@ ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, if (replyRequested && still_sending) { if (reportFlushPosition && lastFlushPosition < blockpos && - walfile != -1) + walfile != NULL) { /* * If a valid flush location needs to be reported, flush the @@ -1109,10 +1011,10 @@ ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, * data has been successfully replicated or not, at the normal * shutdown of the server. */ - if (stream->do_sync && fsync(walfile) != 0) + if (stream->walmethod->fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); + progname, current_walfile_name, stream->walmethod->getlasterror()); return false; } lastFlushPosition = blockpos; @@ -1170,7 +1072,7 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, * Verify that the initial location in the stream matches where we think * we are. */ - if (walfile == -1) + if (walfile == NULL) { /* No file open yet */ if (xlogoff != 0) @@ -1184,12 +1086,11 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, else { /* More data in existing segment */ - /* XXX: store seek value don't reseek all the time */ - if (lseek(walfile, 0, SEEK_CUR) != xlogoff) + if (stream->walmethod->get_current_pos(walfile) != xlogoff) { fprintf(stderr, _("%s: got WAL data offset %08x, expected %08x\n"), - progname, xlogoff, (int) lseek(walfile, 0, SEEK_CUR)); + progname, xlogoff, (int) stream->walmethod->get_current_pos(walfile)); return false; } } @@ -1210,7 +1111,7 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, else bytes_to_write = bytes_left; - if (walfile == -1) + if (walfile == NULL) { if (!open_walfile(stream, *blockpos)) { @@ -1219,14 +1120,13 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, } } - if (write(walfile, - copybuf + hdr_len + bytes_written, - bytes_to_write) != bytes_to_write) + if (stream->walmethod->write(walfile, copybuf + hdr_len + bytes_written, + bytes_to_write) != bytes_to_write) { fprintf(stderr, _("%s: could not write %u bytes to WAL file \"%s\": %s\n"), progname, bytes_to_write, current_walfile_name, - strerror(errno)); + stream->walmethod->getlasterror()); return false; } diff --git a/src/bin/pg_basebackup/receivelog.h b/src/bin/pg_basebackup/receivelog.h index 7a3bbc5080..b5913ea995 100644 --- a/src/bin/pg_basebackup/receivelog.h +++ b/src/bin/pg_basebackup/receivelog.h @@ -13,6 +13,7 @@ #define RECEIVELOG_H #include "libpq-fe.h" +#include "walmethods.h" #include "access/xlogdefs.h" @@ -41,7 +42,7 @@ typedef struct StreamCtl stream_stop_callback stream_stop; /* Stop streaming when returns true */ - char *basedir; /* Received segments written to this dir */ + WalWriteMethod *walmethod; /* How to write the WAL */ char *partial_suffix; /* Suffix appended to partially received files */ } StreamCtl; diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 579d7a15fb..91eb84e238 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -4,7 +4,7 @@ use Config; use PostgresNode; use TestLib; -use Test::More tests => 67; +use Test::More tests => 69; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -237,6 +237,10 @@ 'pg_basebackup -X stream runs'); ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxf/pg_wal")), 'WAL files copied'); +$node->command_ok( + [ 'pg_basebackup', '-D', "$tempdir/backupxst", '-X', 'stream', '-Ft' ], + 'pg_basebackup -X stream runs in tar mode'); +ok(-f "$tempdir/backupxst/pg_wal.tar", "tar file was created"); $node->command_fails( [ 'pg_basebackup', '-D', "$tempdir/fail", '-S', 'slot1' ], diff --git a/src/bin/pg_basebackup/walmethods.c b/src/bin/pg_basebackup/walmethods.c new file mode 100644 index 0000000000..e0ec752bbd --- /dev/null +++ b/src/bin/pg_basebackup/walmethods.c @@ -0,0 +1,886 @@ +/*------------------------------------------------------------------------- + * + * walmethods.c - implementations of different ways to write received wal + * + * NOTE! The caller must ensure that only one method is instantiated in + * any given program, and that it's only instantiated once! + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/walmethods.c + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include +#include +#include +#ifdef HAVE_LIBZ +#include +#endif + +#include "pgtar.h" +#include "common/file_utils.h" + +#include "receivelog.h" +#include "streamutil.h" + +/* Size of zlib buffer for .tar.gz */ +#define ZLIB_OUT_SIZE 4096 + +/*------------------------------------------------------------------------- + * WalDirectoryMethod - write wal to a directory looking like pg_xlog + *------------------------------------------------------------------------- + */ + +/* + * Global static data for this method + */ +typedef struct DirectoryMethodData +{ + char *basedir; + bool sync; +} DirectoryMethodData; +static DirectoryMethodData *dir_data = NULL; + +/* + * Local file handle + */ +typedef struct DirectoryMethodFile +{ + int fd; + off_t currpos; + char *pathname; + char *fullpath; + char *temp_suffix; +} DirectoryMethodFile; + +static char * +dir_getlasterror(void) +{ + /* Directory method always sets errno, so just use strerror */ + return strerror(errno); +} + +static Walfile +dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size) +{ + static char tmppath[MAXPGPATH]; + int fd; + DirectoryMethodFile *f; + + snprintf(tmppath, sizeof(tmppath), "%s/%s%s", + dir_data->basedir, pathname, temp_suffix ? temp_suffix : ""); + + fd = open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) + return NULL; + + if (pad_to_size) + { + /* Always pre-pad on regular files */ + char *zerobuf; + int bytes; + + zerobuf = pg_malloc0(XLOG_BLCKSZ); + for (bytes = 0; bytes < pad_to_size; bytes += XLOG_BLCKSZ) + { + if (write(fd, zerobuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + int save_errno = errno; + + pg_free(zerobuf); + close(fd); + errno = save_errno; + return NULL; + } + } + pg_free(zerobuf); + + if (lseek(fd, 0, SEEK_SET) != 0) + { + int save_errno = errno; + + close(fd); + errno = save_errno; + return NULL; + } + } + + /* + * fsync WAL file and containing directory, to ensure the file is + * persistently created and zeroed (if padded). That's particularly + * important when using synchronous mode, where the file is modified and + * fsynced in-place, without a directory fsync. + */ + if (dir_data->sync) + { + if (fsync_fname(tmppath, false, progname) != 0 || + fsync_parent_path(tmppath, progname) != 0) + { + close(fd); + return NULL; + } + } + + f = pg_malloc0(sizeof(DirectoryMethodFile)); + f->fd = fd; + f->currpos = 0; + f->pathname = pg_strdup(pathname); + f->fullpath = pg_strdup(tmppath); + if (temp_suffix) + f->temp_suffix = pg_strdup(temp_suffix); + + return f; +} + +static ssize_t +dir_write(Walfile f, const void *buf, size_t count) +{ + ssize_t r; + DirectoryMethodFile *df = (DirectoryMethodFile *) f; + + Assert(f != NULL); + + r = write(df->fd, buf, count); + if (r > 0) + df->currpos += r; + return r; +} + +static off_t +dir_get_current_pos(Walfile f) +{ + Assert(f != NULL); + + /* Use a cached value to prevent lots of reseeks */ + return ((DirectoryMethodFile *) f)->currpos; +} + +static int +dir_close(Walfile f, WalCloseMethod method) +{ + int r; + DirectoryMethodFile *df = (DirectoryMethodFile *) f; + static char tmppath[MAXPGPATH]; + static char tmppath2[MAXPGPATH]; + + Assert(f != NULL); + + r = close(df->fd); + + if (r == 0) + { + /* Build path to the current version of the file */ + if (method == CLOSE_NORMAL && df->temp_suffix) + { + /* + * If we have a temp prefix, normal operation is to rename the + * file. + */ + snprintf(tmppath, sizeof(tmppath), "%s/%s%s", + dir_data->basedir, df->pathname, df->temp_suffix); + snprintf(tmppath2, sizeof(tmppath2), "%s/%s", + dir_data->basedir, df->pathname); + r = durable_rename(tmppath, tmppath2, progname); + } + else if (method == CLOSE_UNLINK) + { + /* Unlink the file once it's closed */ + snprintf(tmppath, sizeof(tmppath), "%s/%s%s", + dir_data->basedir, df->pathname, df->temp_suffix ? df->temp_suffix : ""); + r = unlink(tmppath); + } + else + { + /* + * Else either CLOSE_NORMAL and no temp suffix, or + * CLOSE_NO_RENAME. In this case, fsync the file and containing + * directory if sync mode is requested. + */ + if (dir_data->sync) + { + r = fsync_fname(df->fullpath, false, progname); + if (r == 0) + r = fsync_parent_path(df->fullpath, progname); + } + } + } + + pg_free(df->pathname); + pg_free(df->fullpath); + if (df->temp_suffix) + pg_free(df->temp_suffix); + pg_free(df); + + return r; +} + +static int +dir_fsync(Walfile f) +{ + Assert(f != NULL); + + if (!dir_data->sync) + return 0; + + return fsync(((DirectoryMethodFile *) f)->fd); +} + +static ssize_t +dir_get_file_size(const char *pathname) +{ + struct stat statbuf; + static char tmppath[MAXPGPATH]; + + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, pathname); + + if (stat(tmppath, &statbuf) != 0) + return -1; + + return statbuf.st_size; +} + +static bool +dir_existsfile(const char *pathname) +{ + static char tmppath[MAXPGPATH]; + int fd; + + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, pathname); + + fd = open(tmppath, O_RDONLY | PG_BINARY, 0); + if (fd < 0) + return false; + close(fd); + return true; +} + +static bool +dir_finish(void) +{ + if (dir_data->sync) + { + /* + * Files are fsynced when they are closed, but we need to fsync the + * directory entry here as well. + */ + if (fsync_fname(dir_data->basedir, true, progname) != 0) + return false; + } + return true; +} + + +WalWriteMethod * +CreateWalDirectoryMethod(const char *basedir, bool sync) +{ + WalWriteMethod *method; + + method = pg_malloc0(sizeof(WalWriteMethod)); + method->open_for_write = dir_open_for_write; + method->write = dir_write; + method->get_current_pos = dir_get_current_pos; + method->get_file_size = dir_get_file_size; + method->close = dir_close; + method->fsync = dir_fsync; + method->existsfile = dir_existsfile; + method->finish = dir_finish; + method->getlasterror = dir_getlasterror; + + dir_data = pg_malloc0(sizeof(DirectoryMethodData)); + dir_data->basedir = pg_strdup(basedir); + dir_data->sync = sync; + + return method; +} + + +/*------------------------------------------------------------------------- + * WalTarMethod - write wal to a tar file containing pg_xlog contents + *------------------------------------------------------------------------- + */ + +typedef struct TarMethodFile +{ + off_t ofs_start; /* Where does the *header* for this file start */ + off_t currpos; + char header[512]; + char *pathname; + size_t pad_to_size; +} TarMethodFile; + +typedef struct TarMethodData +{ + char *tarfilename; + int fd; + int compression; + bool sync; + TarMethodFile *currentfile; + char lasterror[1024]; +#ifdef HAVE_LIBZ + z_streamp zp; + void *zlibOut; +#endif +} TarMethodData; +static TarMethodData *tar_data = NULL; + +#define tar_clear_error() tar_data->lasterror[0] = '\0' +#define tar_set_error(msg) strlcpy(tar_data->lasterror, msg, sizeof(tar_data->lasterror)) + +static char * +tar_getlasterror(void) +{ + /* + * If a custom error is set, return that one. Otherwise, assume errno is + * set and return that one. + */ + if (tar_data->lasterror[0]) + return tar_data->lasterror; + return strerror(errno); +} + +#ifdef HAVE_LIBZ +static bool +tar_write_compressed_data(void *buf, size_t count, bool flush) +{ + tar_data->zp->next_in = buf; + tar_data->zp->avail_in = count; + + while (tar_data->zp->avail_in || flush) + { + int r; + + r = deflate(tar_data->zp, flush ? Z_FINISH : Z_NO_FLUSH); + if (r == Z_STREAM_ERROR) + { + tar_set_error("deflate failed"); + return false; + } + + if (tar_data->zp->avail_out < ZLIB_OUT_SIZE) + { + size_t len = ZLIB_OUT_SIZE - tar_data->zp->avail_out; + + if (write(tar_data->fd, tar_data->zlibOut, len) != len) + return false; + + tar_data->zp->next_out = tar_data->zlibOut; + tar_data->zp->avail_out = ZLIB_OUT_SIZE; + } + + if (r == Z_STREAM_END) + break; + } + + if (flush) + { + /* Reset the stream for writing */ + if (deflateReset(tar_data->zp) != Z_OK) + { + tar_set_error("deflateReset failed"); + return false; + } + } + + return true; +} +#endif + +static ssize_t +tar_write(Walfile f, const void *buf, size_t count) +{ + ssize_t r; + + Assert(f != NULL); + tar_clear_error(); + + /* Tarfile will always be positioned at the end */ + if (!tar_data->compression) + { + r = write(tar_data->fd, buf, count); + if (r > 0) + ((TarMethodFile *) f)->currpos += r; + return r; + } +#ifdef HAVE_LIBZ + else + { + if (!tar_write_compressed_data((void *) buf, count, false)) + return -1; + ((TarMethodFile *) f)->currpos += count; + return count; + } +#endif +} + +static bool +tar_write_padding_data(TarMethodFile * f, size_t bytes) +{ + char *zerobuf = pg_malloc0(XLOG_BLCKSZ); + size_t bytesleft = bytes; + + while (bytesleft) + { + size_t bytestowrite = bytesleft > XLOG_BLCKSZ ? XLOG_BLCKSZ : bytesleft; + + size_t r = tar_write(f, zerobuf, bytestowrite); + + if (r < 0) + return false; + bytesleft -= r; + } + return true; +} + +static Walfile +tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size) +{ + int save_errno; + static char tmppath[MAXPGPATH]; + + tar_clear_error(); + + if (tar_data->fd < 0) + { + /* + * We open the tar file only when we first try to write to it. + */ + tar_data->fd = open(tar_data->tarfilename, + O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); + if (tar_data->fd < 0) + return NULL; + +#ifdef HAVE_LIBZ + if (tar_data->compression) + { + tar_data->zp = (z_streamp) pg_malloc(sizeof(z_stream)); + tar_data->zp->zalloc = Z_NULL; + tar_data->zp->zfree = Z_NULL; + tar_data->zp->opaque = Z_NULL; + tar_data->zp->next_out = tar_data->zlibOut; + tar_data->zp->avail_out = ZLIB_OUT_SIZE; + + /* + * Initialize deflation library. Adding the magic value 16 to the + * default 15 for the windowBits parameter makes the output be + * gzip instead of zlib. + */ + if (deflateInit2(tar_data->zp, tar_data->compression, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) + { + pg_free(tar_data->zp); + tar_data->zp = NULL; + tar_set_error("deflateInit2 failed"); + return NULL; + } + } +#endif + + /* There's no tar header itself, the file starts with regular files */ + } + + Assert(tar_data->currentfile == NULL); + if (tar_data->currentfile != NULL) + { + tar_set_error("implementation error: tar files can't have more than one open file\n"); + return NULL; + } + + tar_data->currentfile = pg_malloc0(sizeof(TarMethodFile)); + + snprintf(tmppath, sizeof(tmppath), "%s%s", + pathname, temp_suffix ? temp_suffix : ""); + + /* Create a header with size set to 0 - we will fill out the size on close */ + if (tarCreateHeader(tar_data->currentfile->header, tmppath, NULL, 0, S_IRUSR | S_IWUSR, 0, 0, time(NULL)) != TAR_OK) + { + pg_free(tar_data->currentfile); + tar_data->currentfile = NULL; + tar_set_error("could not create tar header"); + return NULL; + } + +#ifdef HAVE_LIBZ + if (tar_data->compression) + { + /* Flush existing data */ + if (!tar_write_compressed_data(NULL, 0, true)) + return NULL; + + /* Turn off compression for header */ + if (deflateParams(tar_data->zp, 0, 0) != Z_OK) + { + tar_set_error("deflateParams failed"); + return NULL; + } + } +#endif + + tar_data->currentfile->ofs_start = lseek(tar_data->fd, 0, SEEK_CUR); + if (tar_data->currentfile->ofs_start == -1) + { + save_errno = errno; + pg_free(tar_data->currentfile); + tar_data->currentfile = NULL; + errno = save_errno; + return NULL; + } + tar_data->currentfile->currpos = 0; + + if (!tar_data->compression) + { + if (write(tar_data->fd, tar_data->currentfile->header, 512) != 512) + { + save_errno = errno; + pg_free(tar_data->currentfile); + tar_data->currentfile = NULL; + errno = save_errno; + return NULL; + } + } +#ifdef HAVE_LIBZ + else + { + /* Write header through the zlib APIs but with no compression */ + if (!tar_write_compressed_data(tar_data->currentfile->header, 512, true)) + return NULL; + + /* Re-enable compression for the rest of the file */ + if (deflateParams(tar_data->zp, tar_data->compression, 0) != Z_OK) + { + tar_set_error("deflateParams failed"); + return NULL; + } + } +#endif + + tar_data->currentfile->pathname = pg_strdup(pathname); + + /* + * Uncompressed files are padded on creation, but for compression we can't + * do that + */ + if (pad_to_size) + { + tar_data->currentfile->pad_to_size = pad_to_size; + if (!tar_data->compression) + { + /* Uncompressed, so pad now */ + tar_write_padding_data(tar_data->currentfile, pad_to_size); + /* Seek back to start */ + if (lseek(tar_data->fd, tar_data->currentfile->ofs_start + 512, SEEK_SET) != tar_data->currentfile->ofs_start + 512) + return NULL; + + tar_data->currentfile->currpos = 0; + } + } + + return tar_data->currentfile; +} + +static ssize_t +tar_get_file_size(const char *pathname) +{ + tar_clear_error(); + + /* Currently not used, so not supported */ + errno = ENOSYS; + return -1; +} + +static off_t +tar_get_current_pos(Walfile f) +{ + Assert(f != NULL); + tar_clear_error(); + + return ((TarMethodFile *) f)->currpos; +} + +static int +tar_fsync(Walfile f) +{ + Assert(f != NULL); + tar_clear_error(); + + /* + * Always sync the whole tarfile, because that's all we can do. This makes + * no sense on compressed files, so just ignore those. + */ + if (tar_data->compression) + return 0; + + return fsync(tar_data->fd); +} + +static int +tar_close(Walfile f, WalCloseMethod method) +{ + ssize_t filesize; + int padding; + TarMethodFile *tf = (TarMethodFile *) f; + + Assert(f != NULL); + tar_clear_error(); + + if (method == CLOSE_UNLINK) + { + if (tar_data->compression) + { + tar_set_error("unlink not supported with compression"); + return -1; + } + + /* + * Unlink the file that we just wrote to the tar. We do this by + * truncating it to the start of the header. This is safe as we only + * allow writing of the very last file. + */ + if (ftruncate(tar_data->fd, tf->ofs_start) != 0) + return -1; + + pg_free(tf->pathname); + pg_free(tf); + tar_data->currentfile = NULL; + + return 0; + } + + /* + * Pad the file itself with zeroes if necessary. Note that this is + * different from the tar format padding -- this is the padding we asked + * for when the file was opened. + */ + if (tf->pad_to_size) + { + if (tar_data->compression) + { + /* + * A compressed tarfile is padded on close since we cannot know + * the size of the compressed output until the end. + */ + size_t sizeleft = tf->pad_to_size - tf->currpos; + + if (sizeleft) + { + if (!tar_write_padding_data(tf, sizeleft)) + return -1; + } + } + else + { + /* + * An uncompressed tarfile was padded on creation, so just adjust + * the current position as if we seeked to the end. + */ + tf->currpos = tf->pad_to_size; + } + } + + /* + * Get the size of the file, and pad the current data up to the nearest + * 512 byte boundary. + */ + filesize = tar_get_current_pos(f); + padding = ((filesize + 511) & ~511) - filesize; + if (padding) + { + char zerobuf[512]; + + MemSet(zerobuf, 0, padding); + if (tar_write(f, zerobuf, padding) != padding) + return -1; + } + + +#ifdef HAVE_LIBZ + if (tar_data->compression) + { + /* Flush the current buffer */ + if (!tar_write_compressed_data(NULL, 0, true)) + { + errno = EINVAL; + return -1; + } + } +#endif + + /* + * Now go back and update the header with the correct filesize and + * possibly also renaming the file. We overwrite the entire current header + * when done, including the checksum. + */ + print_tar_number(&(tf->header[124]), 12, filesize); + + if (method == CLOSE_NORMAL) + + /* + * We overwrite it with what it was before if we have no tempname, + * since we're going to write the buffer anyway. + */ + strlcpy(&(tf->header[0]), tf->pathname, 100); + + print_tar_number(&(tf->header[148]), 8, tarChecksum(((TarMethodFile *) f)->header)); + if (lseek(tar_data->fd, tf->ofs_start, SEEK_SET) != ((TarMethodFile *) f)->ofs_start) + return -1; + if (!tar_data->compression) + { + if (write(tar_data->fd, tf->header, 512) != 512) + return -1; + } +#ifdef HAVE_LIBZ + else + { + /* Turn off compression */ + if (deflateParams(tar_data->zp, 0, 0) != Z_OK) + { + tar_set_error("deflateParams failed"); + return -1; + } + + /* Overwrite the header, assuming the size will be the same */ + if (!tar_write_compressed_data(tar_data->currentfile->header, 512, true)) + return -1; + + /* Turn compression back on */ + if (deflateParams(tar_data->zp, tar_data->compression, 0) != Z_OK) + { + tar_set_error("deflateParams failed"); + return -1; + } + } +#endif + + /* Move file pointer back down to end, so we can write the next file */ + if (lseek(tar_data->fd, 0, SEEK_END) < 0) + return -1; + + /* Always fsync on close, so the padding gets fsynced */ + tar_fsync(f); + + /* Clean up and done */ + pg_free(tf->pathname); + pg_free(tf); + tar_data->currentfile = NULL; + + return 0; +} + +static bool +tar_existsfile(const char *pathname) +{ + tar_clear_error(); + /* We only deal with new tarfiles, so nothing externally created exists */ + return false; +} + +static bool +tar_finish(void) +{ + char zerobuf[1024]; + + tar_clear_error(); + + if (tar_data->currentfile) + { + if (tar_close(tar_data->currentfile, CLOSE_NORMAL) != 0) + return false; + } + + /* A tarfile always ends with two empty blocks */ + MemSet(zerobuf, 0, sizeof(zerobuf)); + if (!tar_data->compression) + { + if (write(tar_data->fd, zerobuf, sizeof(zerobuf)) != sizeof(zerobuf)) + return false; + } +#ifdef HAVE_LIBZ + else + { + if (!tar_write_compressed_data(zerobuf, sizeof(zerobuf), false)) + return false; + + /* Also flush all data to make sure the gzip stream is finished */ + tar_data->zp->next_in = NULL; + tar_data->zp->avail_in = 0; + while (true) + { + int r; + + r = deflate(tar_data->zp, Z_FINISH); + + if (r == Z_STREAM_ERROR) + { + tar_set_error("deflate failed"); + return false; + } + if (tar_data->zp->avail_out < ZLIB_OUT_SIZE) + { + size_t len = ZLIB_OUT_SIZE - tar_data->zp->avail_out; + + if (write(tar_data->fd, tar_data->zlibOut, len) != len) + return false; + } + if (r == Z_STREAM_END) + break; + } + + if (deflateEnd(tar_data->zp) != Z_OK) + { + tar_set_error("deflateEnd failed"); + return false; + } + } +#endif + + /* sync the empty blocks as well, since they're after the last file */ + fsync(tar_data->fd); + + if (close(tar_data->fd) != 0) + return false; + + tar_data->fd = -1; + + if (tar_data->sync) + { + if (fsync_fname(tar_data->tarfilename, false, progname) != 0) + return false; + if (fsync_parent_path(tar_data->tarfilename, progname) != 0) + return false; + } + + return true; +} + +WalWriteMethod * +CreateWalTarMethod(const char *tarbase, int compression, bool sync) +{ + WalWriteMethod *method; + const char *suffix = (compression != 0) ? ".tar.gz" : ".tar"; + + method = pg_malloc0(sizeof(WalWriteMethod)); + method->open_for_write = tar_open_for_write; + method->write = tar_write; + method->get_current_pos = tar_get_current_pos; + method->get_file_size = tar_get_file_size; + method->close = tar_close; + method->fsync = tar_fsync; + method->existsfile = tar_existsfile; + method->finish = tar_finish; + method->getlasterror = tar_getlasterror; + + tar_data = pg_malloc0(sizeof(TarMethodData)); + tar_data->tarfilename = pg_malloc0(strlen(tarbase) + strlen(suffix) + 1); + sprintf(tar_data->tarfilename, "%s%s", tarbase, suffix); + tar_data->fd = -1; + tar_data->compression = compression; + tar_data->sync = sync; + if (compression) + tar_data->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); + + return method; +} diff --git a/src/bin/pg_basebackup/walmethods.h b/src/bin/pg_basebackup/walmethods.h new file mode 100644 index 0000000000..fa58f812f6 --- /dev/null +++ b/src/bin/pg_basebackup/walmethods.h @@ -0,0 +1,45 @@ +/*------------------------------------------------------------------------- + * + * walmethods.h + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/walmethods.h + *------------------------------------------------------------------------- + */ + + +typedef void *Walfile; + +typedef enum +{ + CLOSE_NORMAL, + CLOSE_UNLINK, + CLOSE_NO_RENAME, +} WalCloseMethod; + +typedef struct WalWriteMethod WalWriteMethod; +struct WalWriteMethod +{ + Walfile(*open_for_write) (const char *pathname, const char *temp_suffix, size_t pad_to_size); + int (*close) (Walfile f, WalCloseMethod method); + bool (*existsfile) (const char *pathname); + ssize_t (*get_file_size) (const char *pathname); + + ssize_t (*write) (Walfile f, const void *buf, size_t count); + off_t (*get_current_pos) (Walfile f); + int (*fsync) (Walfile f); + bool (*finish) (void); + char *(*getlasterror) (void); +}; + +/* + * Available WAL methods: + * - WalDirectoryMethod - write WAL to regular files in a standard pg_xlog + * - TarDirectoryMethod - write WAL to a tarfile corresponding to pg_xlog + * (only implements the methods required for pg_basebackup, + * not all those required for pg_receivexlog) + */ +WalWriteMethod *CreateWalDirectoryMethod(const char *basedir, bool sync); +WalWriteMethod *CreateWalTarMethod(const char *tarbase, int compression, bool sync); diff --git a/src/include/pgtar.h b/src/include/pgtar.h index 45ca400f98..1d179f0df1 100644 --- a/src/include/pgtar.h +++ b/src/include/pgtar.h @@ -22,4 +22,5 @@ enum tarError extern enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime); extern uint64 read_tar_number(const char *s, int len); +extern void print_tar_number(char *s, int len, uint64 val); extern int tarChecksum(char *header); diff --git a/src/port/tar.c b/src/port/tar.c index 52a2113a47..f1da959dac 100644 --- a/src/port/tar.c +++ b/src/port/tar.c @@ -16,7 +16,7 @@ * support only non-negative numbers, so we don't worry about the GNU rules * for handling negative numbers.) */ -static void +void print_tar_number(char *s, int len, uint64 val) { if (val < (((uint64) 1) << ((len - 1) * 3))) From d97a59a4c5597af5f53869a5a1c753893752c66b Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sun, 23 Oct 2016 15:56:07 +0200 Subject: [PATCH 365/871] Remove extra comma at end of enum list C99-specific feature, and wasn't intentional in the first place. Per buildfarm member mylodon --- src/bin/pg_basebackup/walmethods.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/pg_basebackup/walmethods.h b/src/bin/pg_basebackup/walmethods.h index fa58f812f6..8a006fda7f 100644 --- a/src/bin/pg_basebackup/walmethods.h +++ b/src/bin/pg_basebackup/walmethods.h @@ -16,7 +16,7 @@ typedef enum { CLOSE_NORMAL, CLOSE_UNLINK, - CLOSE_NO_RENAME, + CLOSE_NO_RENAME } WalCloseMethod; typedef struct WalWriteMethod WalWriteMethod; From 9ae6713cdf32122fa5a0bf15ddcd85f78f6f4631 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sun, 23 Oct 2016 16:00:42 +0200 Subject: [PATCH 366/871] Fix walmethods.c build without libz Per numerous buildfarm manuals --- src/bin/pg_basebackup/walmethods.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bin/pg_basebackup/walmethods.c b/src/bin/pg_basebackup/walmethods.c index e0ec752bbd..d28913fa69 100644 --- a/src/bin/pg_basebackup/walmethods.c +++ b/src/bin/pg_basebackup/walmethods.c @@ -416,6 +416,9 @@ tar_write(Walfile f, const void *buf, size_t count) return count; } #endif + else + /* Can't happen - compression enabled with no libz */ + return -1; } static bool @@ -879,8 +882,10 @@ CreateWalTarMethod(const char *tarbase, int compression, bool sync) tar_data->fd = -1; tar_data->compression = compression; tar_data->sync = sync; +#ifdef HAVE_LIBZ if (compression) tar_data->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); +#endif return method; } From a5c17c1dcef4a656559152d3f6a5a27ae4957843 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sun, 23 Oct 2016 16:07:31 +0200 Subject: [PATCH 367/871] Fix obviously too quickly applied fix to zlib issue --- src/bin/pg_basebackup/walmethods.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_basebackup/walmethods.c b/src/bin/pg_basebackup/walmethods.c index d28913fa69..fe6569dff3 100644 --- a/src/bin/pg_basebackup/walmethods.c +++ b/src/bin/pg_basebackup/walmethods.c @@ -415,10 +415,11 @@ tar_write(Walfile f, const void *buf, size_t count) ((TarMethodFile *) f)->currpos += count; return count; } -#endif +#else else /* Can't happen - compression enabled with no libz */ return -1; +#endif } static bool From eade082b122889eaf92eb806b8b6799160a25256 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sun, 23 Oct 2016 18:04:34 +0200 Subject: [PATCH 368/871] Rename walmethod fsync method to sync Using the name fsync clashed with the #define we have on Windows that redefines it to _commit. Naming it sync should remove that conflict. Per all the Windows buildfarm members --- src/bin/pg_basebackup/receivelog.c | 6 +++--- src/bin/pg_basebackup/walmethods.c | 10 +++++----- src/bin/pg_basebackup/walmethods.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index fcd0269473..4382e5d76a 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -132,7 +132,7 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) } /* fsync file in case of a previous crash */ - if (!stream->walmethod->fsync(f)) + if (!stream->walmethod->sync(f)) { stream->walmethod->close(f, CLOSE_UNLINK); return false; @@ -768,7 +768,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, */ if (stream->synchronous && lastFlushPosition < blockpos && walfile != NULL) { - if (stream->walmethod->fsync(walfile) != 0) + if (stream->walmethod->sync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, stream->walmethod->getlasterror()); @@ -1011,7 +1011,7 @@ ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, * data has been successfully replicated or not, at the normal * shutdown of the server. */ - if (stream->walmethod->fsync(walfile) != 0) + if (stream->walmethod->sync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, stream->walmethod->getlasterror()); diff --git a/src/bin/pg_basebackup/walmethods.c b/src/bin/pg_basebackup/walmethods.c index fe6569dff3..d1dc046c97 100644 --- a/src/bin/pg_basebackup/walmethods.c +++ b/src/bin/pg_basebackup/walmethods.c @@ -219,7 +219,7 @@ dir_close(Walfile f, WalCloseMethod method) } static int -dir_fsync(Walfile f) +dir_sync(Walfile f) { Assert(f != NULL); @@ -287,7 +287,7 @@ CreateWalDirectoryMethod(const char *basedir, bool sync) method->get_current_pos = dir_get_current_pos; method->get_file_size = dir_get_file_size; method->close = dir_close; - method->fsync = dir_fsync; + method->sync = dir_sync; method->existsfile = dir_existsfile; method->finish = dir_finish; method->getlasterror = dir_getlasterror; @@ -606,7 +606,7 @@ tar_get_current_pos(Walfile f) } static int -tar_fsync(Walfile f) +tar_sync(Walfile f) { Assert(f != NULL); tar_clear_error(); @@ -764,7 +764,7 @@ tar_close(Walfile f, WalCloseMethod method) return -1; /* Always fsync on close, so the padding gets fsynced */ - tar_fsync(f); + tar_sync(f); /* Clean up and done */ pg_free(tf->pathname); @@ -872,7 +872,7 @@ CreateWalTarMethod(const char *tarbase, int compression, bool sync) method->get_current_pos = tar_get_current_pos; method->get_file_size = tar_get_file_size; method->close = tar_close; - method->fsync = tar_fsync; + method->sync = tar_sync; method->existsfile = tar_existsfile; method->finish = tar_finish; method->getlasterror = tar_getlasterror; diff --git a/src/bin/pg_basebackup/walmethods.h b/src/bin/pg_basebackup/walmethods.h index 8a006fda7f..0c8eac7c61 100644 --- a/src/bin/pg_basebackup/walmethods.h +++ b/src/bin/pg_basebackup/walmethods.h @@ -29,7 +29,7 @@ struct WalWriteMethod ssize_t (*write) (Walfile f, const void *buf, size_t count); off_t (*get_current_pos) (Walfile f); - int (*fsync) (Walfile f); + int (*sync) (Walfile f); bool (*finish) (void); char *(*getlasterror) (void); }; From 6292c2339186bac215bab5a1f01370f9735582c1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 23 Oct 2016 15:01:24 -0400 Subject: [PATCH 369/871] Avoid testing tuple visibility without buffer lock in RI_FKey_check(). Despite the argumentation I wrote in commit 7a2fe85b0, it's unsafe to do this, because in corner cases it's possible for HeapTupleSatisfiesSelf to try to set hint bits on the target tuple; and at least since 8.2 we have required the buffer content lock to be held while setting hint bits. The added regression test exercises one such corner case. Unpatched, it causes an assertion failure in assert-enabled builds, or otherwise would cause a hint bit change in a buffer we don't hold lock on, which given the right race condition could result in checksum failures or other data consistency problems. The odds of a problem in the field are probably pretty small, but nonetheless back-patch to all supported branches. Report: <19391.1477244876@sss.pgh.pa.us> --- src/backend/utils/adt/ri_triggers.c | 22 ++++++++++------------ src/test/regress/expected/foreign_key.out | 21 +++++++++++++++++++++ src/test/regress/sql/foreign_key.sql | 23 +++++++++++++++++++++++ 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/backend/utils/adt/ri_triggers.c b/src/backend/utils/adt/ri_triggers.c index b4765005fe..983f7647d7 100644 --- a/src/backend/utils/adt/ri_triggers.c +++ b/src/backend/utils/adt/ri_triggers.c @@ -44,6 +44,7 @@ #include "parser/parse_coerce.h" #include "parser/parse_relation.h" #include "miscadmin.h" +#include "storage/bufmgr.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -286,20 +287,17 @@ RI_FKey_check(TriggerData *trigdata) * We should not even consider checking the row if it is no longer valid, * since it was either deleted (so the deferred check should be skipped) * or updated (in which case only the latest version of the row should be - * checked). Test its liveness according to SnapshotSelf. - * - * NOTE: The normal coding rule is that one must acquire the buffer - * content lock to call HeapTupleSatisfiesVisibility. We can skip that - * here because we know that AfterTriggerExecute just fetched the tuple - * successfully, so there cannot be a VACUUM compaction in progress on the - * page (either heap_fetch would have waited for the VACUUM, or the - * VACUUM's LockBufferForCleanup would be waiting for us to drop pin). And - * since this is a row inserted by our open transaction, no one else can - * be entitled to change its xmin/xmax. - */ - Assert(new_row_buf != InvalidBuffer); + * checked). Test its liveness according to SnapshotSelf. We need pin + * and lock on the buffer to call HeapTupleSatisfiesVisibility. Caller + * should be holding pin, but not lock. + */ + LockBuffer(new_row_buf, BUFFER_LOCK_SHARE); if (!HeapTupleSatisfiesVisibility(new_row, SnapshotSelf, new_row_buf)) + { + LockBuffer(new_row_buf, BUFFER_LOCK_UNLOCK); return PointerGetDatum(NULL); + } + LockBuffer(new_row_buf, BUFFER_LOCK_UNLOCK); /* * Get the relation descriptors of the FK and PK tables. diff --git a/src/test/regress/expected/foreign_key.out b/src/test/regress/expected/foreign_key.out index 044881af71..46a30b4428 100644 --- a/src/test/regress/expected/foreign_key.out +++ b/src/test/regress/expected/foreign_key.out @@ -1381,3 +1381,24 @@ explain (costs off) delete from t1 where a = 1; (10 rows) delete from t1 where a = 1; +-- +-- Test deferred FK check on a tuple deleted by a rolled-back subtransaction +-- +create table pktable2(f1 int primary key); +create table fktable2(f1 int references pktable2 deferrable initially deferred); +insert into pktable2 values(1); +begin; +insert into fktable2 values(1); +savepoint x; +delete from fktable2; +rollback to x; +commit; +begin; +insert into fktable2 values(2); +savepoint x; +delete from fktable2; +rollback to x; +commit; -- fail +ERROR: insert or update on table "fktable2" violates foreign key constraint "fktable2_f1_fkey" +DETAIL: Key (f1)=(2) is not present in table "pktable2". +drop table pktable2, fktable2; diff --git a/src/test/regress/sql/foreign_key.sql b/src/test/regress/sql/foreign_key.sql index 85c9d04d64..cc36ab5baf 100644 --- a/src/test/regress/sql/foreign_key.sql +++ b/src/test/regress/sql/foreign_key.sql @@ -1019,3 +1019,26 @@ create rule r1 as on delete to t1 do delete from t2 where t2.b = old.a; explain (costs off) delete from t1 where a = 1; delete from t1 where a = 1; + +-- +-- Test deferred FK check on a tuple deleted by a rolled-back subtransaction +-- +create table pktable2(f1 int primary key); +create table fktable2(f1 int references pktable2 deferrable initially deferred); +insert into pktable2 values(1); + +begin; +insert into fktable2 values(1); +savepoint x; +delete from fktable2; +rollback to x; +commit; + +begin; +insert into fktable2 values(2); +savepoint x; +delete from fktable2; +rollback to x; +commit; -- fail + +drop table pktable2, fktable2; From a6c0a5b6e8a9498540c6a7bb1b6d68958acc9bc6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 23 Oct 2016 18:36:13 -0400 Subject: [PATCH 370/871] Don't throw serialization errors for self-conflicts in INSERT ON CONFLICT. A transaction that conflicts against itself, for example INSERT INTO t(pk) VALUES (1),(1) ON CONFLICT DO NOTHING; should behave the same regardless of isolation level. It certainly shouldn't throw a serialization error, as retrying will not help. We got this wrong due to the ON CONFLICT logic not considering the case, as reported by Jason Dusek. Core of this patch is by Peter Geoghegan (based on an earlier patch by Thomas Munro), though I didn't take his proposed code refactoring for fear that it might have unexpected side-effects. Test cases by Thomas Munro and myself. Report: Related-Discussion: <57EE93C8.8080504@postgrespro.ru> --- src/backend/executor/nodeModifyTable.c | 13 ++- .../expected/insert-conflict-do-nothing-2.out | 105 ++++++++++++++++++ src/test/isolation/isolation_schedule | 1 + .../specs/insert-conflict-do-nothing-2.spec | 34 ++++++ src/test/regress/expected/insert_conflict.out | 35 ++++++ src/test/regress/sql/insert_conflict.sql | 32 ++++++ 6 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 src/test/isolation/expected/insert-conflict-do-nothing-2.out create mode 100644 src/test/isolation/specs/insert-conflict-do-nothing-2.spec diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index af7b26c0ef..b056dd9e95 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -195,9 +195,18 @@ ExecCheckHeapTupleVisible(EState *estate, return; if (!HeapTupleSatisfiesVisibility(tuple, estate->es_snapshot, buffer)) - ereport(ERROR, - (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), + { + /* + * We should not raise a serialization failure if the conflict is + * against a tuple inserted by our own transaction, even if it's not + * visible to our snapshot. (This would happen, for example, if + * conflicting keys are proposed for insertion in a single command.) + */ + if (!TransactionIdIsCurrentTransactionId(HeapTupleHeaderGetXmin(tuple->t_data))) + ereport(ERROR, + (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); + } } /* diff --git a/src/test/isolation/expected/insert-conflict-do-nothing-2.out b/src/test/isolation/expected/insert-conflict-do-nothing-2.out new file mode 100644 index 0000000000..2332f96978 --- /dev/null +++ b/src/test/isolation/expected/insert-conflict-do-nothing-2.out @@ -0,0 +1,105 @@ +Parsed test spec with 2 sessions + +starting permutation: beginrr1 beginrr2 donothing1 c1 donothing2 c2 show +step beginrr1: BEGIN ISOLATION LEVEL REPEATABLE READ; +step beginrr2: BEGIN ISOLATION LEVEL REPEATABLE READ; +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step c1: COMMIT; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2'), (1, 'donothing3') ON CONFLICT DO NOTHING; +step c2: COMMIT; +step show: SELECT * FROM ints; +key val + +1 donothing1 + +starting permutation: beginrr1 beginrr2 donothing2 c2 donothing1 c1 show +step beginrr1: BEGIN ISOLATION LEVEL REPEATABLE READ; +step beginrr2: BEGIN ISOLATION LEVEL REPEATABLE READ; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2'), (1, 'donothing3') ON CONFLICT DO NOTHING; +step c2: COMMIT; +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step c1: COMMIT; +step show: SELECT * FROM ints; +key val + +1 donothing2 + +starting permutation: beginrr1 beginrr2 donothing1 donothing2 c1 c2 show +step beginrr1: BEGIN ISOLATION LEVEL REPEATABLE READ; +step beginrr2: BEGIN ISOLATION LEVEL REPEATABLE READ; +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2'), (1, 'donothing3') ON CONFLICT DO NOTHING; +step c1: COMMIT; +step donothing2: <... completed> +error in steps c1 donothing2: ERROR: could not serialize access due to concurrent update +step c2: COMMIT; +step show: SELECT * FROM ints; +key val + +1 donothing1 + +starting permutation: beginrr1 beginrr2 donothing2 donothing1 c2 c1 show +step beginrr1: BEGIN ISOLATION LEVEL REPEATABLE READ; +step beginrr2: BEGIN ISOLATION LEVEL REPEATABLE READ; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2'), (1, 'donothing3') ON CONFLICT DO NOTHING; +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step c2: COMMIT; +step donothing1: <... completed> +error in steps c2 donothing1: ERROR: could not serialize access due to concurrent update +step c1: COMMIT; +step show: SELECT * FROM ints; +key val + +1 donothing2 + +starting permutation: begins1 begins2 donothing1 c1 donothing2 c2 show +step begins1: BEGIN ISOLATION LEVEL SERIALIZABLE; +step begins2: BEGIN ISOLATION LEVEL SERIALIZABLE; +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step c1: COMMIT; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2'), (1, 'donothing3') ON CONFLICT DO NOTHING; +step c2: COMMIT; +step show: SELECT * FROM ints; +key val + +1 donothing1 + +starting permutation: begins1 begins2 donothing2 c2 donothing1 c1 show +step begins1: BEGIN ISOLATION LEVEL SERIALIZABLE; +step begins2: BEGIN ISOLATION LEVEL SERIALIZABLE; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2'), (1, 'donothing3') ON CONFLICT DO NOTHING; +step c2: COMMIT; +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step c1: COMMIT; +step show: SELECT * FROM ints; +key val + +1 donothing2 + +starting permutation: begins1 begins2 donothing1 donothing2 c1 c2 show +step begins1: BEGIN ISOLATION LEVEL SERIALIZABLE; +step begins2: BEGIN ISOLATION LEVEL SERIALIZABLE; +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2'), (1, 'donothing3') ON CONFLICT DO NOTHING; +step c1: COMMIT; +step donothing2: <... completed> +error in steps c1 donothing2: ERROR: could not serialize access due to concurrent update +step c2: COMMIT; +step show: SELECT * FROM ints; +key val + +1 donothing1 + +starting permutation: begins1 begins2 donothing2 donothing1 c2 c1 show +step begins1: BEGIN ISOLATION LEVEL SERIALIZABLE; +step begins2: BEGIN ISOLATION LEVEL SERIALIZABLE; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2'), (1, 'donothing3') ON CONFLICT DO NOTHING; +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step c2: COMMIT; +step donothing1: <... completed> +error in steps c2 donothing1: ERROR: could not serialize access due to concurrent update +step c1: COMMIT; +step show: SELECT * FROM ints; +key val + +1 donothing2 diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index a96a318987..2606a27624 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -25,6 +25,7 @@ test: eval-plan-qual test: lock-update-delete test: lock-update-traversal test: insert-conflict-do-nothing +test: insert-conflict-do-nothing-2 test: insert-conflict-do-update test: insert-conflict-do-update-2 test: insert-conflict-do-update-3 diff --git a/src/test/isolation/specs/insert-conflict-do-nothing-2.spec b/src/test/isolation/specs/insert-conflict-do-nothing-2.spec new file mode 100644 index 0000000000..f1e5bde357 --- /dev/null +++ b/src/test/isolation/specs/insert-conflict-do-nothing-2.spec @@ -0,0 +1,34 @@ +# INSERT...ON CONFLICT DO NOTHING test with multiple rows +# in higher isolation levels + +setup +{ + CREATE TABLE ints (key int primary key, val text); +} + +teardown +{ + DROP TABLE ints; +} + +session "s1" +step "beginrr1" { BEGIN ISOLATION LEVEL REPEATABLE READ; } +step "begins1" { BEGIN ISOLATION LEVEL SERIALIZABLE; } +step "donothing1" { INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; } +step "c1" { COMMIT; } +step "show" { SELECT * FROM ints; } + +session "s2" +step "beginrr2" { BEGIN ISOLATION LEVEL REPEATABLE READ; } +step "begins2" { BEGIN ISOLATION LEVEL SERIALIZABLE; } +step "donothing2" { INSERT INTO ints(key, val) VALUES(1, 'donothing2'), (1, 'donothing3') ON CONFLICT DO NOTHING; } +step "c2" { COMMIT; } + +permutation "beginrr1" "beginrr2" "donothing1" "c1" "donothing2" "c2" "show" +permutation "beginrr1" "beginrr2" "donothing2" "c2" "donothing1" "c1" "show" +permutation "beginrr1" "beginrr2" "donothing1" "donothing2" "c1" "c2" "show" +permutation "beginrr1" "beginrr2" "donothing2" "donothing1" "c2" "c1" "show" +permutation "begins1" "begins2" "donothing1" "c1" "donothing2" "c2" "show" +permutation "begins1" "begins2" "donothing2" "c2" "donothing1" "c1" "show" +permutation "begins1" "begins2" "donothing1" "donothing2" "c1" "c2" "show" +permutation "begins1" "begins2" "donothing2" "donothing1" "c2" "c1" "show" diff --git a/src/test/regress/expected/insert_conflict.out b/src/test/regress/expected/insert_conflict.out index 155774ecfa..8d8d69b1ad 100644 --- a/src/test/regress/expected/insert_conflict.out +++ b/src/test/regress/expected/insert_conflict.out @@ -727,3 +727,38 @@ select * from twoconstraints; (1 row) drop table twoconstraints; +-- check handling of self-conflicts at various isolation levels +create table selfconflict (f1 int primary key, f2 int); +begin transaction isolation level read committed; +insert into selfconflict values (1,1), (1,2) on conflict do nothing; +commit; +begin transaction isolation level repeatable read; +insert into selfconflict values (2,1), (2,2) on conflict do nothing; +commit; +begin transaction isolation level serializable; +insert into selfconflict values (3,1), (3,2) on conflict do nothing; +commit; +begin transaction isolation level read committed; +insert into selfconflict values (4,1), (4,2) on conflict(f1) do update set f2 = 0; +ERROR: ON CONFLICT DO UPDATE command cannot affect row a second time +HINT: Ensure that no rows proposed for insertion within the same command have duplicate constrained values. +commit; +begin transaction isolation level repeatable read; +insert into selfconflict values (5,1), (5,2) on conflict(f1) do update set f2 = 0; +ERROR: ON CONFLICT DO UPDATE command cannot affect row a second time +HINT: Ensure that no rows proposed for insertion within the same command have duplicate constrained values. +commit; +begin transaction isolation level serializable; +insert into selfconflict values (6,1), (6,2) on conflict(f1) do update set f2 = 0; +ERROR: ON CONFLICT DO UPDATE command cannot affect row a second time +HINT: Ensure that no rows proposed for insertion within the same command have duplicate constrained values. +commit; +select * from selfconflict; + f1 | f2 +----+---- + 1 | 1 + 2 | 1 + 3 | 1 +(3 rows) + +drop table selfconflict; diff --git a/src/test/regress/sql/insert_conflict.sql b/src/test/regress/sql/insert_conflict.sql index 190c605562..81c4a7ca4b 100644 --- a/src/test/regress/sql/insert_conflict.sql +++ b/src/test/regress/sql/insert_conflict.sql @@ -421,3 +421,35 @@ insert into twoconstraints values(2, '((0,0),(1,2))') on conflict on constraint twoconstraints_f2_excl do nothing; -- do nothing select * from twoconstraints; drop table twoconstraints; + +-- check handling of self-conflicts at various isolation levels + +create table selfconflict (f1 int primary key, f2 int); + +begin transaction isolation level read committed; +insert into selfconflict values (1,1), (1,2) on conflict do nothing; +commit; + +begin transaction isolation level repeatable read; +insert into selfconflict values (2,1), (2,2) on conflict do nothing; +commit; + +begin transaction isolation level serializable; +insert into selfconflict values (3,1), (3,2) on conflict do nothing; +commit; + +begin transaction isolation level read committed; +insert into selfconflict values (4,1), (4,2) on conflict(f1) do update set f2 = 0; +commit; + +begin transaction isolation level repeatable read; +insert into selfconflict values (5,1), (5,2) on conflict(f1) do update set f2 = 0; +commit; + +begin transaction isolation level serializable; +insert into selfconflict values (6,1), (6,2) on conflict(f1) do update set f2 = 0; +commit; + +select * from selfconflict; + +drop table selfconflict; From 8f1fb7d621b0e6bd2eb0ba2ac9634c5b5a03564b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 23 Oct 2016 19:14:32 -0400 Subject: [PATCH 371/871] Avoid testing tuple visibility without buffer lock. INSERT ... ON CONFLICT (specifically ExecCheckHeapTupleVisible) contains another example of this unsafe coding practice. It is much harder to get a failure out of it than the case fixed in commit 6292c2339, because in most scenarios any hint bits that could be set would have already been set earlier in the command. However, Konstantin Knizhnik reported a failure with a custom transaction manager, and it's clearly possible to get a failure via a race condition in async-commit mode. For lack of a reproducible example, no regression test case in this commit. I did some testing with Asserts added to tqual.c's functions, and can say that running "make check-world" exposed these two bugs and no others. The Asserts are messy enough that I've not added them to the code for now. Report: <57EE93C8.8080504@postgrespro.ru> Related-Discussion: --- src/backend/executor/nodeModifyTable.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index b056dd9e95..efb0c5e8e5 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -194,6 +194,11 @@ ExecCheckHeapTupleVisible(EState *estate, if (!IsolationUsesXactSnapshot()) return; + /* + * We need buffer pin and lock to call HeapTupleSatisfiesVisibility. + * Caller should be holding pin, but not lock. + */ + LockBuffer(buffer, BUFFER_LOCK_SHARE); if (!HeapTupleSatisfiesVisibility(tuple, estate->es_snapshot, buffer)) { /* @@ -207,6 +212,7 @@ ExecCheckHeapTupleVisible(EState *estate, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); } + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); } /* From 7d80417d3dfc88b0c03b5c08a18b29f9d430e217 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 23 Oct 2016 22:13:28 -0400 Subject: [PATCH 372/871] Release notes for 9.6.1, 9.5.5, 9.4.10, 9.3.15, 9.2.19, 9.1.24. --- doc/src/sgml/release-9.1.sgml | 210 +++++++++ doc/src/sgml/release-9.2.sgml | 266 +++++++++++ doc/src/sgml/release-9.3.sgml | 329 ++++++++++++++ doc/src/sgml/release-9.4.sgml | 466 +++++++++++++++++++ doc/src/sgml/release-9.5.sgml | 812 ++++++++++++++++++++++++++++++++++ doc/src/sgml/release-9.6.sgml | 586 +----------------------- 6 files changed, 2103 insertions(+), 566 deletions(-) diff --git a/doc/src/sgml/release-9.1.sgml b/doc/src/sgml/release-9.1.sgml index 26b709056f..edacfbf355 100644 --- a/doc/src/sgml/release-9.1.sgml +++ b/doc/src/sgml/release-9.1.sgml @@ -1,6 +1,216 @@ + + Release 9.1.24 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.1.23. + For information about new features in the 9.1 major release, see + . + + + + This is expected to be the last PostgreSQL release + in the 9.1.X series. Users are encouraged to update to a newer + release branch soon. + + + + Migration to Version 9.1.24 + + + A dump/restore is not required for those running 9.1.X. + + + + However, if you are upgrading from a version earlier than 9.1.16, + see . + + + + + + Changes + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + Make ecpg's + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.1.23 diff --git a/doc/src/sgml/release-9.2.sgml b/doc/src/sgml/release-9.2.sgml index 0f6e3d127f..49430389d9 100644 --- a/doc/src/sgml/release-9.2.sgml +++ b/doc/src/sgml/release-9.2.sgml @@ -1,6 +1,272 @@ + + Release 9.2.19 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.2.18. + For information about new features in the 9.2 major release, see + . + + + + Migration to Version 9.2.19 + + + A dump/restore is not required for those running 9.2.X. + + + + However, if you are upgrading from a version earlier than 9.2.11, + see . + + + + + + Changes + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + Fix EXPLAIN to emit valid XML when + is on (Markus Winand) + + + + Previously the XML output-format option produced syntactically invalid + tags such as <I/O-Read-Time>. That is now + rendered as <I-O-Read-Time>. + + + + + + Suppress printing of zeroes for unmeasured times + in EXPLAIN (Maksim Milyutin) + + + + Certain option combinations resulted in printing zero values for times + that actually aren't ever measured in that combination. Our general + policy in EXPLAIN is not to print such fields at all, so + do that consistently in all cases. + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Fix bugs in merging inherited CHECK constraints while + creating or altering a table (Tom Lane, Amit Langote) + + + + Allow identical CHECK constraints to be added to a parent + and child table in either order. Prevent merging of a valid + constraint from the parent table with a NOT VALID + constraint on the child. Likewise, prevent merging of a NO + INHERIT child constraint with an inherited constraint. + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + Make ecpg's + + + + + In pg_dump, never dump range constructor functions + (Tom Lane) + + + + This oversight led to pg_upgrade failures with + extensions containing range types, due to duplicate creation of the + constructor functions. + + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + Update Windows time zone mapping to recognize some time zone names + added in recent Windows versions (Michael Paquier) + + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.2.18 diff --git a/doc/src/sgml/release-9.3.sgml b/doc/src/sgml/release-9.3.sgml index e321f4b31c..81205a40c7 100644 --- a/doc/src/sgml/release-9.3.sgml +++ b/doc/src/sgml/release-9.3.sgml @@ -1,6 +1,335 @@ + + Release 9.3.15 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.3.14. + For information about new features in the 9.3 major release, see + . + + + + Migration to Version 9.3.15 + + + A dump/restore is not required for those running 9.3.X. + + + + However, if your installation has been affected by the bug described in + the first changelog entry below, then after updating you may need + to take action to repair corrupted free space maps. + + + + Also, if you are upgrading from a version earlier than 9.3.9, + see . + + + + + + Changes + + + + + + Fix WAL-logging of truncation of relation free space maps and + visibility maps (Pavan Deolasee, Heikki Linnakangas) + + + + It was possible for these files to not be correctly restored during + crash recovery, or to be written incorrectly on a standby server. + Bogus entries in a free space map could lead to attempts to access + pages that have been truncated away from the relation itself, typically + producing errors like could not read block XXX: + read only 0 of 8192 bytes. Checksum failures in the + visibility map are also possible, if checksumming is enabled. + + + + Procedures for determining whether there is a problem and repairing it + if so are discussed at + . + + + + + + Fix SELECT FOR UPDATE/SHARE to correctly lock tuples that + have been updated by a subsequently-aborted transaction + (Álvaro Herrera) + + + + In 9.5 and later, the SELECT would sometimes fail to + return such tuples at all. A failure has not been proven to occur in + earlier releases, but might be possible with concurrent updates. + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + Fix EXPLAIN to emit valid XML when + is on (Markus Winand) + + + + Previously the XML output-format option produced syntactically invalid + tags such as <I/O-Read-Time>. That is now + rendered as <I-O-Read-Time>. + + + + + + Suppress printing of zeroes for unmeasured times + in EXPLAIN (Maksim Milyutin) + + + + Certain option combinations resulted in printing zero values for times + that actually aren't ever measured in that combination. Our general + policy in EXPLAIN is not to print such fields at all, so + do that consistently in all cases. + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Fix bugs in merging inherited CHECK constraints while + creating or altering a table (Tom Lane, Amit Langote) + + + + Allow identical CHECK constraints to be added to a parent + and child table in either order. Prevent merging of a valid + constraint from the parent table with a NOT VALID + constraint on the child. Likewise, prevent merging of a NO + INHERIT child constraint with an inherited constraint. + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + Make ecpg's + + + + + In pg_dump, never dump range constructor functions + (Tom Lane) + + + + This oversight led to pg_upgrade failures with + extensions containing range types, due to duplicate creation of the + constructor functions. + + + + + + In pg_xlogdump, retry opening new WAL segments when + using + + + This allows for a possible delay in the server's creation of the next + segment. + + + + + + Fix pg_xlogdump to cope with a WAL file that begins + with a continuation record spanning more than one page (Pavan + Deolasee) + + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + Update Windows time zone mapping to recognize some time zone names + added in recent Windows versions (Michael Paquier) + + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.3.14 diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index 51896924c9..94b028a065 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -1,6 +1,472 @@ + + Release 9.4.10 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.4.9. + For information about new features in the 9.4 major release, see + . + + + + Migration to Version 9.4.10 + + + A dump/restore is not required for those running 9.4.X. + + + + However, if your installation has been affected by the bug described in + the first changelog entry below, then after updating you may need + to take action to repair corrupted free space maps. + + + + Also, if you are upgrading from a version earlier than 9.4.6, + see . + + + + + Changes + + + + + + Fix WAL-logging of truncation of relation free space maps and + visibility maps (Pavan Deolasee, Heikki Linnakangas) + + + + It was possible for these files to not be correctly restored during + crash recovery, or to be written incorrectly on a standby server. + Bogus entries in a free space map could lead to attempts to access + pages that have been truncated away from the relation itself, typically + producing errors like could not read block XXX: + read only 0 of 8192 bytes. Checksum failures in the + visibility map are also possible, if checksumming is enabled. + + + + Procedures for determining whether there is a problem and repairing it + if so are discussed at + . + + + + + + Fix incorrect creation of GIN index WAL records on big-endian machines + (Tom Lane) + + + + The typical symptom was unexpected GIN leaf action errors + during WAL replay. + + + + + + Fix SELECT FOR UPDATE/SHARE to correctly lock tuples that + have been updated by a subsequently-aborted transaction + (Álvaro Herrera) + + + + In 9.5 and later, the SELECT would sometimes fail to + return such tuples at all. A failure has not been proven to occur in + earlier releases, but might be possible with concurrent updates. + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + Fix query-lifespan memory leak in a bulk UPDATE on a table + with a PRIMARY KEY or REPLICA IDENTITY index + (Tom Lane) + + + + + + Fix EXPLAIN to emit valid XML when + is on (Markus Winand) + + + + Previously the XML output-format option produced syntactically invalid + tags such as <I/O-Read-Time>. That is now + rendered as <I-O-Read-Time>. + + + + + + Suppress printing of zeroes for unmeasured times + in EXPLAIN (Maksim Milyutin) + + + + Certain option combinations resulted in printing zero values for times + that actually aren't ever measured in that combination. Our general + policy in EXPLAIN is not to print such fields at all, so + do that consistently in all cases. + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Fix bugs in merging inherited CHECK constraints while + creating or altering a table (Tom Lane, Amit Langote) + + + + Allow identical CHECK constraints to be added to a parent + and child table in either order. Prevent merging of a valid + constraint from the parent table with a NOT VALID + constraint on the child. Likewise, prevent merging of a NO + INHERIT child constraint with an inherited constraint. + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix logical WAL decoding to work properly when a subtransaction's WAL + output is large enough to spill to disk (Andres Freund) + + + + + + + Fix buffer overread in logical WAL decoding (Tom Lane) + + + + Logical decoding of a tuple update record read 23 bytes too many, + which was usually harmless but with very bad luck could result in a + crash. + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + Properly initialize replication slot state when recycling a + previously-used slot (Michael Paquier) + + + + This failure to reset all of the fields of the slot could + prevent VACUUM from removing dead tuples. + + + + + + Round shared-memory allocation request to a multiple of the actual + huge page size when attempting to use huge pages on Linux (Tom Lane) + + + + This avoids possible failures during munmap() on systems + with atypical default huge page sizes. Except in crash-recovery + cases, there were no ill effects other than a log message. + + + + + + Use a more random value for the dynamic shared memory control + segment's ID (Robert Haas, Tom Lane) + + + + Previously, the same value would be chosen every time, because it was + derived from random() but srandom() had not + yet been called. While relatively harmless, this was not the intended + behavior. + + + + + + On Windows, retry creation of the dynamic shared memory control + segment after an access-denied error (Kyotaro Horiguchi, Amit Kapila) + + + + Windows sometimes returns ERROR_ACCESS_DENIED rather + than ERROR_ALREADY_EXISTS when there is an existing + segment. This led to postmaster startup failure due to believing that + the former was an unrecoverable error. + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + Make ecpg's + + + + + Fix pgbench's calculation of average latency + (Fabien Coelho) + + + + The calculation was incorrect when there were \sleep + commands in the script, or when the test duration was specified in + number of transactions rather than total time. + + + + + + In pg_dump, never dump range constructor functions + (Tom Lane) + + + + This oversight led to pg_upgrade failures with + extensions containing range types, due to duplicate creation of the + constructor functions. + + + + + + In pg_xlogdump, retry opening new WAL segments when + using + + + This allows for a possible delay in the server's creation of the next + segment. + + + + + + Fix pg_xlogdump to cope with a WAL file that begins + with a continuation record spanning more than one page (Pavan + Deolasee) + + + + + + Fix contrib/pg_buffercache to work + when shared_buffers exceeds 256GB (KaiGai Kohei) + + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + Install TAP test infrastructure so that it's available for extension + testing (Craig Ringer) + + + + When PostgreSQL has been configured + with + + + + + In MSVC builds, include pg_recvlogical in a + client-only installation (MauMau) + + + + + + Update Windows time zone mapping to recognize some time zone names + added in recent Windows versions (Michael Paquier) + + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.4.9 diff --git a/doc/src/sgml/release-9.5.sgml b/doc/src/sgml/release-9.5.sgml index c3f0f7051e..2102b1dd29 100644 --- a/doc/src/sgml/release-9.5.sgml +++ b/doc/src/sgml/release-9.5.sgml @@ -1,6 +1,818 @@ + + Release 9.5.5 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.5.4. + For information about new features in the 9.5 major release, see + . + + + + Migration to Version 9.5.5 + + + A dump/restore is not required for those running 9.5.X. + + + + However, if your installation has been affected by the bug described in + the first changelog entry below, then after updating you may need + to take action to repair corrupted free space maps. + + + + Also, if you are upgrading from a version earlier than 9.5.2, + see . + + + + + Changes + + + + + + Fix WAL-logging of truncation of relation free space maps and + visibility maps (Pavan Deolasee, Heikki Linnakangas) + + + + It was possible for these files to not be correctly restored during + crash recovery, or to be written incorrectly on a standby server. + Bogus entries in a free space map could lead to attempts to access + pages that have been truncated away from the relation itself, typically + producing errors like could not read block XXX: + read only 0 of 8192 bytes. Checksum failures in the + visibility map are also possible, if checksumming is enabled. + + + + Procedures for determining whether there is a problem and repairing it + if so are discussed at + . + + + + + + + Fix incorrect creation of GIN index WAL records on big-endian machines + (Tom Lane) + + + + The typical symptom was unexpected GIN leaf action errors + during WAL replay. + + + + + + + Fix SELECT FOR UPDATE/SHARE to correctly lock tuples that + have been updated by a subsequently-aborted transaction + (Álvaro Herrera) + + + + In 9.5 and later, the SELECT would sometimes fail to + return such tuples at all. A failure has not been proven to occur in + earlier releases, but might be possible with concurrent updates. + + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + + Fix deletion of speculatively inserted TOAST tuples when backing out + of INSERT ... ON CONFLICT (Oskari Saarenmaa) + + + + In the race condition where two transactions try to insert conflicting + tuples at about the same time, the loser would fail with + an attempted to delete invisible tuple error if its + insertion included any TOAST'ed fields. + + + + + + Don't throw serialization errors for self-conflicting insertions + in INSERT ... ON CONFLICT (Thomas Munro, Peter Geoghegan) + + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + + Fix query-lifespan memory leak in a bulk UPDATE on a table + with a PRIMARY KEY or REPLICA IDENTITY index + (Tom Lane) + + + + + + Fix COPY with a column name list from a table that has + row-level security enabled (Adam Brightwell) + + + + + + Fix EXPLAIN to emit valid XML when + is on (Markus Winand) + + + + Previously the XML output-format option produced syntactically invalid + tags such as <I/O-Read-Time>. That is now + rendered as <I-O-Read-Time>. + + + + + + + Suppress printing of zeroes for unmeasured times + in EXPLAIN (Maksim Milyutin) + + + + Certain option combinations resulted in printing zero values for times + that actually aren't ever measured in that combination. Our general + policy in EXPLAIN is not to print such fields at all, so + do that consistently in all cases. + + + + + + Fix statistics update for TRUNCATE in a prepared + transaction (Stas Kelvich) + + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Fix bugs in merging inherited CHECK constraints while + creating or altering a table (Tom Lane, Amit Langote) + + + + Allow identical CHECK constraints to be added to a parent + and child table in either order. Prevent merging of a valid + constraint from the parent table with a NOT VALID + constraint on the child. Likewise, prevent merging of a NO + INHERIT child constraint with an inherited constraint. + + + + + + Show a sensible value + in pg_settings.unit + for min_wal_size and max_wal_size (Tom Lane) + + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix logical WAL decoding to work properly when a subtransaction's WAL + output is large enough to spill to disk (Andres Freund) + + + + + + + Fix possible sorting error when aborting use of abbreviated keys + (Peter Geoghegan) + + + + In the worst case, this could result in a corrupt btree index, which + would need to be rebuilt using REINDEX. However, the + situation is believed to be rare. + + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + + Properly initialize replication slot state when recycling a + previously-used slot (Michael Paquier) + + + + This failure to reset all of the fields of the slot could + prevent VACUUM from removing dead tuples. + + + + + + Round shared-memory allocation request to a multiple of the actual + huge page size when attempting to use huge pages on Linux (Tom Lane) + + + + This avoids possible failures during munmap() on systems + with atypical default huge page sizes. Except in crash-recovery + cases, there were no ill effects other than a log message. + + + + + + + Use a more random value for the dynamic shared memory control + segment's ID (Robert Haas, Tom Lane) + + + + Previously, the same value would be chosen every time, because it was + derived from random() but srandom() had not + yet been called. While relatively harmless, this was not the intended + behavior. + + + + + + + On Windows, retry creation of the dynamic shared memory control + segment after an access-denied error (Kyotaro Horiguchi, Amit Kapila) + + + + Windows sometimes returns ERROR_ACCESS_DENIED rather + than ERROR_ALREADY_EXISTS when there is an existing + segment. This led to postmaster startup failure due to believing that + the former was an unrecoverable error. + + + + + + + Fix PL/pgSQL to not misbehave with parameters and + local variables of type int2vector or oidvector + (Tom Lane) + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + + Make ecpg's + + + + + + Fix pgbench's calculation of average latency + (Fabien Coelho) + + + + The calculation was incorrect when there were \sleep + commands in the script, or when the test duration was specified in + number of transactions rather than total time. + + + + + + In pg_upgrade, check library loadability in name order + (Tom Lane) + + + + This is a workaround to deal with cross-extension dependencies from + language transform modules to their base language and data type + modules. + + + + + + + In pg_dump, never dump range constructor functions + (Tom Lane) + + + + This oversight led to pg_upgrade failures with + extensions containing range types, due to duplicate creation of the + constructor functions. + + + + + + + In pg_dump with + + + + + + Make pg_receivexlog work correctly + with + + + + + Disallow specifying both + + + + + Make pg_rewind turn off synchronous_commit + in its session on the source server (Michael Banck, Michael Paquier) + + + + This allows pg_rewind to work even when the source + server is using synchronous replication that is not working for some + reason. + + + + + + In pg_xlogdump, retry opening new WAL segments when + using + + + This allows for a possible delay in the server's creation of the next + segment. + + + + + + + Fix pg_xlogdump to cope with a WAL file that begins + with a continuation record spanning more than one page (Pavan + Deolasee) + + + + + + + Fix contrib/pg_buffercache to work + when shared_buffers exceeds 256GB (KaiGai Kohei) + + + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + + Support OpenSSL 1.1.0 (Heikki Linnakangas) + + + + + + + Install TAP test infrastructure so that it's available for extension + testing (Craig Ringer) + + + + When PostgreSQL has been configured + with + + + + + + In MSVC builds, include pg_recvlogical in a + client-only installation (MauMau) + + + + + + + Update Windows time zone mapping to recognize some time zone names + added in recent Windows versions (Michael Paquier) + + + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.5.4 diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index ebdeda4445..8b3f51428d 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -62,7 +62,7 @@ Branch: REL9_3_STABLE [1c02ee314] 2016-10-19 15:00:34 +0300 Procedures for determining whether there is a problem and repairing it if so are discussed at - . + . @@ -96,89 +96,20 @@ Branch: REL9_6_STABLE [b6d906073] 2016-09-30 20:39:06 -0400 with contrib/pg_visibility's pg_truncate_visibility_map() function. For more information see - . - - - - - - - Fix incorrect creation of GIN index WAL records on big-endian machines - (Tom Lane) - - - - The typical symptom was unexpected GIN leaf action errors - during WAL replay. - - - - - - - Fix SELECT FOR UPDATE/SHARE to correctly lock tuples that - have been updated by a subsequently-aborted transaction - (Álvaro Herrera) - - - - In 9.5 and later, the SELECT would sometimes fail to - return such tuples at all. A failure has not been proven to occur in - earlier releases, but might be possible with concurrent updates. + . - Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) - - - - The recheck would always see the CTE as returning no rows, typically - leading to failure to update rows that were recently updated. - - - - - - - Fix deletion of speculatively inserted TOAST tuples when backing out - of INSERT ... ON CONFLICT (Oskari Saarenmaa) - - - - In the race condition where two transactions try to insert conflicting - tuples at about the same time, the loser would fail with - an attempted to delete invisible tuple error if its - insertion included any TOAST'ed fields. + Don't throw serialization errors for self-conflicting insertions + in INSERT ... ON CONFLICT (Thomas Munro, Peter Geoghegan) @@ -219,46 +150,6 @@ Branch: REL9_6_STABLE [dca25c256] 2016-10-09 12:49:37 -0400 - - Fix improper repetition of previous results from hashed aggregation in - a subquery (Andrew Gierth) - - - - The test to see if we can reuse a previously-computed hash table of - the aggregate state values neglected the possibility of an outer query - reference appearing in an aggregate argument expression. A change in - the value of such a reference should lead to recalculating the hash - table, but did not. - - - - - - - Fix query-lifespan memory leak in a bulk UPDATE on a table - with a PRIMARY KEY or REPLICA IDENTITY index - (Tom Lane) - - - - - - - Suppress printing of zeroes for unmeasured times - in EXPLAIN (Maksim Milyutin) - - - - Certain option combinations resulted in printing zero values for times - that actually aren't ever measured in that combination. Our general - policy in EXPLAIN is not to print such fields at all, so - do that consistently in all cases. - - - - - - - Fix timeout length when VACUUM is waiting for exclusive - table lock so that it can truncate the table (Simon Riggs) - - - - The timeout was meant to be 50 milliseconds, but it was actually only - 50 microseconds, causing VACUUM to give up on truncation - much more easily than intended. Set it to the intended value. - - - - - - Remove artificial restrictions on the values accepted - by numeric_in() and numeric_recv() - (Tom Lane) - - - - We allow numeric values up to the limit of the storage format (more - than 1e100000), so it seems fairly pointless - that numeric_in() rejected scientific-notation exponents - above 1000. Likewise, it was silly for numeric_recv() to - reject more than 1000 digits in an input value. + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) @@ -464,100 +306,6 @@ Branch: REL9_6_STABLE [32841fa32] 2016-09-28 11:22:39 -0400 - - Fix buffer overread in logical WAL decoding (Tom Lane) - - - - Logical decoding of a tuple update record read 23 bytes too many, - which was usually harmless but with very bad luck could result in a - crash. - - - - - - - Fix possible sorting error when aborting use of abbreviated keys - (Peter Geoghegan) - - - - In the worst case, this could result in a corrupt btree index, which - would need to be rebuilt using REINDEX. However, the - situation is believed to be rare. - - - - - - - Fix file descriptor leakage when truncating a temporary relation of - more than 1GB (Andres Freund) - - - - - - - Disallow starting a standalone backend with standby_mode - turned on (Michael Paquier) - - - - This can't do anything useful, since there will be no WAL receiver - process to fetch more WAL data; and it could result in misbehavior - in code that wasn't designed with this situation in mind. - - - - - - - Properly initialize replication slot state when recycling a - previously-used slot (Michael Paquier) - - - - This failure to reset all of the fields of the slot could - prevent VACUUM from removing dead tuples. - - - - - - - Use a more random value for the dynamic shared memory control - segment's ID (Robert Haas, Tom Lane) - - - - Previously, the same value would be chosen every time, because it was - derived from random() but srandom() had not - yet been called. While relatively harmless, this was not the intended - behavior. - - - - - - - On Windows, retry creation of the dynamic shared memory control - segment after an access-denied error (Kyotaro Horiguchi, Amit Kapila) - - - - Windows sometimes returns ERROR_ACCESS_DENIED rather - than ERROR_ALREADY_EXISTS when there is an existing - segment. This led to postmaster startup failure due to believing that - the former was an unrecoverable error. - - - - - - - Fix PL/pgSQL to not misbehave with parameters and - local variables of type int2vector or oidvector - (Tom Lane) - - - - - - - Make ecpg's - - - - - - Fix pgbench's calculation of average latency - (Fabien Coelho) - - - - The calculation was incorrect when there were \sleep - commands in the script, or when the test duration was specified in - number of transactions rather than total time. - - - - - - - In pg_dump, never dump range constructor functions - (Tom Lane) - - - - This oversight led to pg_upgrade failures with - extensions containing range types, due to duplicate creation of the - constructor functions. - - - - - - - In pg_dump with - - - - @@ -827,20 +437,6 @@ Branch: REL9_6_STABLE [1749332ec] 2016-10-07 09:51:28 -0400 - - Make pg_receivexlog work correctly - with - - - - - - Fix pg_xlogdump to cope with a WAL file that begins - with a continuation record spanning more than one page (Pavan - Deolasee) - - - - - - - Fix contrib/pg_buffercache to work - when shared_buffers exceeds 256GB (KaiGai Kohei) - - - - - - - Fix contrib/intarray/bench/bench.pl to print the results - of the EXPLAIN it does when given the - - - - - - Support OpenSSL 1.1.0 (Heikki Linnakangas) - - - - - - - Install TAP test infrastructure so that it's available for extension - testing (Craig Ringer) - - - - When PostgreSQL has been configured - with - - - - - - In MSVC builds, include pg_recvlogical in a - client-only installation (MauMau) - - - - - - - Update Windows time zone mapping to recognize some time zone names - added in recent Windows versions (Michael Paquier) - - - - - - - Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) - - - - If a dynamic time zone abbreviation does not match any entry in the - referenced time zone, treat it as equivalent to the time zone name. - This avoids unexpected failures when IANA removes abbreviations from - their time zone database, as they did in tzdata - release 2016f and seem likely to do again in the future. The - consequences were not limited to not recognizing the individual - abbreviation; any mismatch caused - the pg_timezone_abbrevs view to fail altogether. - - - - - + + Preserve commit timestamps across server restart + (Julien Rouhaud, Craig Ringer) + + + + With turned on, old + commit timestamps became inaccessible after a clean server restart. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + +
+

+ + + +

+
+ + + + + + + +
+
+
+ + + + + +
+

+ + + +

+
+ + + + + + + +
+
+
+
+
+ + + + + + + + +
+
+ + + + + + + + + +
+ + + + + + +

+ +

+
+
+ + + + + + + +
+
+
+
+ + + + + + + + + + + + | + + + + + + + + + + From 7016e4c4f59c30d22f308dd29e8a5014d6427f69 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 4 Nov 2016 09:03:42 -0400 Subject: [PATCH 424/871] postgres_fdw: Fix typo in comment. Etsuro Fujita --- contrib/postgres_fdw/postgres_fdw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 2cfb82bf59..fbe69295e7 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -4156,7 +4156,7 @@ foreign_join_ok(PlannerInfo *root, RelOptInfo *joinrel, JoinType jointype, * other remote clauses. For LEFT and RIGHT OUTER join, the clauses from * the outer side are added to remote_conds since those can be evaluated * after the join is evaluated. The clauses from inner side are added to - * the joinclauses, since they need to evaluated while constructing the + * the joinclauses, since they need to be evaluated while constructing the * join. * * For a FULL OUTER JOIN, the other clauses from either relation can not From f2e6a2ccf1247bc15f6c03895c936b28cdeff181 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 4 Nov 2016 09:27:49 -0400 Subject: [PATCH 425/871] Add API to check if an existing exclusive lock allows cleanup. LockBufferForCleanup() acquires a cleanup lock unconditionally, and ConditionalLockBufferForCleanup() acquires a cleanup lock if it is possible to do so without waiting; this patch adds a new API, IsBufferCleanupOK(), which tests whether an exclusive lock already held happens to be a cleanup lock. This is possible because a cleanup lock simply means an exclusive lock plus the assurance any other pins on the buffer are newer than our own pin. Therefore, just as the existing functions decide that the exclusive lock that they've just taken is a cleanup lock if they observe the pin count to be 1, this new function allows us to observe that the pin count is 1 on a buffer we've already locked. This is useful in situations where a backend definitely wishes to modify the buffer and also wishes to perform cleanup operations if possible. The patch to eliminate heavyweight locking by hash indexes uses this, and it may have other applications as well. Amit Kapila, per a suggestion from me. Some comment adjustments by me as well. --- src/backend/storage/buffer/bufmgr.c | 49 +++++++++++++++++++++++++++++ src/include/storage/bufmgr.h | 1 + 2 files changed, 50 insertions(+) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index df4c9d7109..58b0a975c0 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -3745,6 +3745,55 @@ ConditionalLockBufferForCleanup(Buffer buffer) return false; } +/* + * IsBufferCleanupOK - as above, but we already have the lock + * + * Check whether it's OK to perform cleanup on a buffer we've already + * locked. If we observe that the pin count is 1, our exclusive lock + * happens to be a cleanup lock, and we can proceed with anything that + * would have been allowable had we sought a cleanup lock originally. + */ +bool +IsBufferCleanupOK(Buffer buffer) +{ + BufferDesc *bufHdr; + uint32 buf_state; + + Assert(BufferIsValid(buffer)); + + if (BufferIsLocal(buffer)) + { + /* There should be exactly one pin */ + if (LocalRefCount[-buffer - 1] != 1) + return false; + /* Nobody else to wait for */ + return true; + } + + /* There should be exactly one local pin */ + if (GetPrivateRefCount(buffer) != 1) + return false; + + bufHdr = GetBufferDescriptor(buffer - 1); + + /* caller must hold exclusive lock on buffer */ + Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), + LW_EXCLUSIVE)); + + buf_state = LockBufHdr(bufHdr); + + Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); + if (BUF_STATE_GET_REFCOUNT(buf_state) == 1) + { + /* pincount is OK. */ + UnlockBufHdr(bufHdr, buf_state); + return true; + } + + UnlockBufHdr(bufHdr, buf_state); + return false; +} + /* * Functions for buffer I/O handling diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 7b6ba96000..821bee5ece 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -227,6 +227,7 @@ extern void LockBuffer(Buffer buffer, int mode); extern bool ConditionalLockBuffer(Buffer buffer); extern void LockBufferForCleanup(Buffer buffer); extern bool ConditionalLockBufferForCleanup(Buffer buffer); +extern bool IsBufferCleanupOK(Buffer buffer); extern bool HoldingBufferPinThatDelaysRecovery(void); extern void AbortBufferIO(void); From 84ad68d645222f534e4fe51fbba43a5c9cc382a5 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 4 Nov 2016 12:00:00 -0400 Subject: [PATCH 426/871] pageinspect: Fix unaligned struct access in GIN functions The raw page data that is passed into the functions will not be aligned at 8-byte boundaries. Casting that to a struct and accessing int64 fields will result in unaligned access. On most platforms, you get away with it, but it will result on a crash on pickier platforms such as ia64 and sparc64. --- contrib/pageinspect/ginfuncs.c | 46 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/contrib/pageinspect/ginfuncs.c b/contrib/pageinspect/ginfuncs.c index 145e2ce709..6e4103df6a 100644 --- a/contrib/pageinspect/ginfuncs.c +++ b/contrib/pageinspect/ginfuncs.c @@ -28,11 +28,31 @@ PG_FUNCTION_INFO_V1(gin_metapage_info); PG_FUNCTION_INFO_V1(gin_page_opaque_info); PG_FUNCTION_INFO_V1(gin_leafpage_items); + +static Page +get_page_from_raw(bytea *raw_page) +{ + int raw_page_size; + Page page; + + raw_page_size = VARSIZE(raw_page) - VARHDRSZ; + if (raw_page_size < BLCKSZ) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("input page too small (%d bytes)", raw_page_size))); + + /* make a copy so that the page is properly aligned for struct access */ + page = palloc(raw_page_size); + memcpy(page, VARDATA(raw_page), raw_page_size); + + return page; +} + + Datum gin_metapage_info(PG_FUNCTION_ARGS) { bytea *raw_page = PG_GETARG_BYTEA_P(0); - int raw_page_size; TupleDesc tupdesc; Page page; GinPageOpaque opaq; @@ -46,12 +66,7 @@ gin_metapage_info(PG_FUNCTION_ARGS) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be superuser to use raw page functions")))); - raw_page_size = VARSIZE(raw_page) - VARHDRSZ; - if (raw_page_size < BLCKSZ) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("input page too small (%d bytes)", raw_page_size))); - page = VARDATA(raw_page); + page = get_page_from_raw(raw_page); opaq = (GinPageOpaque) PageGetSpecialPointer(page); if (opaq->flags != GIN_META) @@ -94,7 +109,6 @@ Datum gin_page_opaque_info(PG_FUNCTION_ARGS) { bytea *raw_page = PG_GETARG_BYTEA_P(0); - int raw_page_size; TupleDesc tupdesc; Page page; GinPageOpaque opaq; @@ -110,12 +124,7 @@ gin_page_opaque_info(PG_FUNCTION_ARGS) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be superuser to use raw page functions")))); - raw_page_size = VARSIZE(raw_page) - VARHDRSZ; - if (raw_page_size < BLCKSZ) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("input page too small (%d bytes)", raw_page_size))); - page = VARDATA(raw_page); + page = get_page_from_raw(raw_page); opaq = (GinPageOpaque) PageGetSpecialPointer(page); @@ -173,7 +182,6 @@ Datum gin_leafpage_items(PG_FUNCTION_ARGS) { bytea *raw_page = PG_GETARG_BYTEA_P(0); - int raw_page_size; FuncCallContext *fctx; gin_leafpage_items_state *inter_call_data; @@ -182,8 +190,6 @@ gin_leafpage_items(PG_FUNCTION_ARGS) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be superuser to use raw page functions")))); - raw_page_size = VARSIZE(raw_page) - VARHDRSZ; - if (SRF_IS_FIRSTCALL()) { TupleDesc tupdesc; @@ -191,11 +197,7 @@ gin_leafpage_items(PG_FUNCTION_ARGS) Page page; GinPageOpaque opaq; - if (raw_page_size < BLCKSZ) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("input page too small (%d bytes)", raw_page_size))); - page = VARDATA(raw_page); + page = get_page_from_raw(raw_page); if (PageGetSpecialSize(page) != MAXALIGN(sizeof(GinPageOpaqueData))) ereport(ERROR, From 632fbe772cff9dadca82a26dacaa39bd50a0cc5c Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 14 Oct 2016 12:00:00 -0400 Subject: [PATCH 427/871] pg_archivecleanup: Add NLS Reviewed-by: Michael Paquier --- src/bin/pg_archivecleanup/nls.mk | 4 + src/bin/pg_archivecleanup/pg_archivecleanup.c | 73 ++++++++++--------- 2 files changed, 41 insertions(+), 36 deletions(-) create mode 100644 src/bin/pg_archivecleanup/nls.mk diff --git a/src/bin/pg_archivecleanup/nls.mk b/src/bin/pg_archivecleanup/nls.mk new file mode 100644 index 0000000000..fd959a5c8b --- /dev/null +++ b/src/bin/pg_archivecleanup/nls.mk @@ -0,0 +1,4 @@ +# src/bin/pg_archivecleanup/nls.mk +CATALOG_NAME = pg_archivecleanup +AVAIL_LANGUAGES = +GETTEXT_FILES = pg_archivecleanup.c diff --git a/src/bin/pg_archivecleanup/pg_archivecleanup.c b/src/bin/pg_archivecleanup/pg_archivecleanup.c index 319038fc80..f1651d4273 100644 --- a/src/bin/pg_archivecleanup/pg_archivecleanup.c +++ b/src/bin/pg_archivecleanup/pg_archivecleanup.c @@ -70,7 +70,7 @@ Initialize(void) if (stat(archiveLocation, &stat_buf) != 0 || !S_ISDIR(stat_buf.st_mode)) { - fprintf(stderr, "%s: archive location \"%s\" does not exist\n", + fprintf(stderr, _("%s: archive location \"%s\" does not exist\n"), progname, archiveLocation); exit(2); } @@ -146,19 +146,19 @@ CleanupPriorWALFiles(void) printf("%s\n", WALFilePath); if (debug) fprintf(stderr, - "%s: file \"%s\" would be removed\n", + _("%s: file \"%s\" would be removed\n"), progname, WALFilePath); continue; } if (debug) - fprintf(stderr, "%s: removing file \"%s\"\n", + fprintf(stderr, _("%s: removing file \"%s\"\n"), progname, WALFilePath); rc = unlink(WALFilePath); if (rc != 0) { - fprintf(stderr, "%s: ERROR: could not remove file \"%s\": %s\n", + fprintf(stderr, _("%s: ERROR: could not remove file \"%s\": %s\n"), progname, WALFilePath, strerror(errno)); break; } @@ -166,14 +166,14 @@ CleanupPriorWALFiles(void) } if (errno) - fprintf(stderr, "%s: could not read archive location \"%s\": %s\n", + fprintf(stderr, _("%s: could not read archive location \"%s\": %s\n"), progname, archiveLocation, strerror(errno)); if (closedir(xldir)) - fprintf(stderr, "%s: could not close archive location \"%s\": %s\n", + fprintf(stderr, _("%s: could not close archive location \"%s\": %s\n"), progname, archiveLocation, strerror(errno)); } else - fprintf(stderr, "%s: could not open archive location \"%s\": %s\n", + fprintf(stderr, _("%s: could not open archive location \"%s\": %s\n"), progname, archiveLocation, strerror(errno)); } @@ -246,8 +246,8 @@ SetWALFileNameForCleanup(void) if (!fnameOK) { - fprintf(stderr, "%s: invalid filename input\n", progname); - fprintf(stderr, "Try \"%s --help\" for more information.\n", progname); + fprintf(stderr, _("%s: invalid filename input\n"), progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(2); } } @@ -260,25 +260,25 @@ SetWALFileNameForCleanup(void) static void usage(void) { - printf("%s removes older WAL files from PostgreSQL archives.\n\n", progname); - printf("Usage:\n"); - printf(" %s [OPTION]... ARCHIVELOCATION OLDESTKEPTWALFILE\n", progname); - printf("\nOptions:\n"); - printf(" -d generate debug output (verbose mode)\n"); - printf(" -n dry run, show the names of the files that would be removed\n"); - printf(" -V, --version output version information, then exit\n"); - printf(" -x EXT clean up files if they have this extension\n"); - printf(" -?, --help show this help, then exit\n"); - printf("\n" - "For use as archive_cleanup_command in recovery.conf when standby_mode = on:\n" - " archive_cleanup_command = 'pg_archivecleanup [OPTION]... ARCHIVELOCATION %%r'\n" - "e.g.\n" - " archive_cleanup_command = 'pg_archivecleanup /mnt/server/archiverdir %%r'\n"); - printf("\n" - "Or for use as a standalone archive cleaner:\n" - "e.g.\n" - " pg_archivecleanup /mnt/server/archiverdir 000000010000000000000010.00000020.backup\n"); - printf("\nReport bugs to .\n"); + printf(_("%s removes older WAL files from PostgreSQL archives.\n\n"), progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]... ARCHIVELOCATION OLDESTKEPTWALFILE\n"), progname); + printf(_("\nOptions:\n")); + printf(_(" -d generate debug output (verbose mode)\n")); + printf(_(" -n dry run, show the names of the files that would be removed\n")); + printf(_(" -V, --version output version information, then exit\n")); + printf(_(" -x EXT clean up files if they have this extension\n")); + printf(_(" -?, --help show this help, then exit\n")); + printf(_("\n" + "For use as archive_cleanup_command in recovery.conf when standby_mode = on:\n" + " archive_cleanup_command = 'pg_archivecleanup [OPTION]... ARCHIVELOCATION %%r'\n" + "e.g.\n" + " archive_cleanup_command = 'pg_archivecleanup /mnt/server/archiverdir %%r'\n")); + printf(_("\n" + "Or for use as a standalone archive cleaner:\n" + "e.g.\n" + " pg_archivecleanup /mnt/server/archiverdir 000000010000000000000010.00000020.backup\n")); + printf(_("\nReport bugs to .\n")); } /*------------ MAIN ----------------------------------------*/ @@ -287,6 +287,7 @@ main(int argc, char **argv) { int c; + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_archivecleanup")); progname = get_progname(argv[0]); if (argc > 1) @@ -318,7 +319,7 @@ main(int argc, char **argv) * from xlogfile names */ break; default: - fprintf(stderr, "Try \"%s --help\" for more information.\n", progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(2); break; } @@ -338,8 +339,8 @@ main(int argc, char **argv) } else { - fprintf(stderr, "%s: must specify archive location\n", progname); - fprintf(stderr, "Try \"%s --help\" for more information.\n", progname); + fprintf(stderr, _("%s: must specify archive location\n"), progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(2); } @@ -350,15 +351,15 @@ main(int argc, char **argv) } else { - fprintf(stderr, "%s: must specify restartfilename\n", progname); - fprintf(stderr, "Try \"%s --help\" for more information.\n", progname); + fprintf(stderr, _("%s: must specify restartfilename\n"), progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(2); } if (optind < argc) { - fprintf(stderr, "%s: too many parameters\n", progname); - fprintf(stderr, "Try \"%s --help\" for more information.\n", progname); + fprintf(stderr, _("%s: too many parameters\n"), progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(2); } @@ -376,7 +377,7 @@ main(int argc, char **argv) { snprintf(WALFilePath, MAXPGPATH, "%s/%s", archiveLocation, exclusiveCleanupFileName); - fprintf(stderr, "%s: keep WAL file \"%s\" and later\n", + fprintf(stderr, _("%s: keep WAL file \"%s\" and later\n"), progname, WALFilePath); } From a39255d766381957622b145838c38dbbea6fc367 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 14 Oct 2016 12:00:00 -0400 Subject: [PATCH 428/871] pg_test_fsync: Add NLS Reviewed-by: Michael Paquier --- src/bin/pg_test_fsync/nls.mk | 5 ++ src/bin/pg_test_fsync/pg_test_fsync.c | 71 ++++++++++++++------------- 2 files changed, 41 insertions(+), 35 deletions(-) create mode 100644 src/bin/pg_test_fsync/nls.mk diff --git a/src/bin/pg_test_fsync/nls.mk b/src/bin/pg_test_fsync/nls.mk new file mode 100644 index 0000000000..6c95731020 --- /dev/null +++ b/src/bin/pg_test_fsync/nls.mk @@ -0,0 +1,5 @@ +# src/bin/pg_test_fsync/nls.mk +CATALOG_NAME = pg_test_fsync +AVAIL_LANGUAGES = +GETTEXT_FILES = pg_test_fsync.c +GETTEXT_TRIGGERS = die diff --git a/src/bin/pg_test_fsync/pg_test_fsync.c b/src/bin/pg_test_fsync/pg_test_fsync.c index c8427623d2..d65c0ab110 100644 --- a/src/bin/pg_test_fsync/pg_test_fsync.c +++ b/src/bin/pg_test_fsync/pg_test_fsync.c @@ -44,7 +44,7 @@ do { \ if (CreateThread(NULL, 0, process_alarm, NULL, 0, NULL) == \ INVALID_HANDLE_VALUE) \ { \ - fprintf(stderr, "Cannot create thread for alarm\n"); \ + fprintf(stderr, _("Cannot create thread for alarm\n")); \ exit(1); \ } \ gettimeofday(&start_t, NULL); \ @@ -96,6 +96,7 @@ static void die(const char *str); int main(int argc, char *argv[]) { + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_test_fsync")); progname = get_progname(argv[0]); handle_args(argc, argv); @@ -148,7 +149,7 @@ handle_args(int argc, char *argv[]) { if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) { - printf("Usage: %s [-f FILENAME] [-s SECS-PER-TEST]\n", progname); + printf(_("Usage: %s [-f FILENAME] [-s SECS-PER-TEST]\n"), progname); exit(0); } if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) @@ -172,7 +173,7 @@ handle_args(int argc, char *argv[]) break; default: - fprintf(stderr, "Try \"%s --help\" for more information.\n", + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); break; @@ -182,18 +183,18 @@ handle_args(int argc, char *argv[]) if (argc > optind) { fprintf(stderr, - "%s: too many command-line arguments (first is \"%s\")\n", + _("%s: too many command-line arguments (first is \"%s\")\n"), progname, argv[optind]); - fprintf(stderr, "Try \"%s --help\" for more information.\n", + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } - printf("%d seconds per test\n", secs_per_test); + printf(_("%d seconds per test\n"), secs_per_test); #if PG_O_DIRECT != 0 - printf("O_DIRECT supported on this platform for open_datasync and open_sync.\n"); + printf(_("O_DIRECT supported on this platform for open_datasync and open_sync.\n")); #else - printf("Direct I/O is not supported on this platform.\n"); + printf(_("Direct I/O is not supported on this platform.\n")); #endif } @@ -239,10 +240,10 @@ test_sync(int writes_per_op) bool fs_warning = false; if (writes_per_op == 1) - printf("\nCompare file sync methods using one %dkB write:\n", XLOG_BLCKSZ_K); + printf(_("\nCompare file sync methods using one %dkB write:\n"), XLOG_BLCKSZ_K); else - printf("\nCompare file sync methods using two %dkB writes:\n", XLOG_BLCKSZ_K); - printf("(in wal_sync_method preference order, except fdatasync is Linux's default)\n"); + printf(_("\nCompare file sync methods using two %dkB writes:\n"), XLOG_BLCKSZ_K); + printf(_("(in wal_sync_method preference order, except fdatasync is Linux's default)\n")); /* * Test open_datasync if available @@ -253,7 +254,7 @@ test_sync(int writes_per_op) #ifdef OPEN_DATASYNC_FLAG if ((tmpfile = open(filename, O_RDWR | O_DSYNC | PG_O_DIRECT, 0)) == -1) { - printf(NA_FORMAT, "n/a*\n"); + printf(NA_FORMAT, _("n/a*\n")); fs_warning = true; } else @@ -271,7 +272,7 @@ test_sync(int writes_per_op) close(tmpfile); } #else - printf(NA_FORMAT, "n/a\n"); + printf(NA_FORMAT, _("n/a\n")); #endif /* @@ -296,7 +297,7 @@ test_sync(int writes_per_op) STOP_TIMER; close(tmpfile); #else - printf(NA_FORMAT, "n/a\n"); + printf(NA_FORMAT, _("n/a\n")); #endif /* @@ -344,7 +345,7 @@ test_sync(int writes_per_op) STOP_TIMER; close(tmpfile); #else - printf(NA_FORMAT, "n/a\n"); + printf(NA_FORMAT, _("n/a\n")); #endif /* @@ -356,7 +357,7 @@ test_sync(int writes_per_op) #ifdef OPEN_SYNC_FLAG if ((tmpfile = open(filename, O_RDWR | OPEN_SYNC_FLAG | PG_O_DIRECT, 0)) == -1) { - printf(NA_FORMAT, "n/a*\n"); + printf(NA_FORMAT, _("n/a*\n")); fs_warning = true; } else @@ -381,28 +382,28 @@ test_sync(int writes_per_op) close(tmpfile); } #else - printf(NA_FORMAT, "n/a\n"); + printf(NA_FORMAT, _("n/a\n")); #endif if (fs_warning) { - printf("* This file system and its mount options do not support direct\n"); - printf("I/O, e.g. ext4 in journaled mode.\n"); + printf(_("* This file system and its mount options do not support direct\n" + " I/O, e.g. ext4 in journaled mode.\n")); } } static void test_open_syncs(void) { - printf("\nCompare open_sync with different write sizes:\n"); - printf("(This is designed to compare the cost of writing 16kB in different write\n" - "open_sync sizes.)\n"); - - test_open_sync(" 1 * 16kB open_sync write", 16); - test_open_sync(" 2 * 8kB open_sync writes", 8); - test_open_sync(" 4 * 4kB open_sync writes", 4); - test_open_sync(" 8 * 2kB open_sync writes", 2); - test_open_sync("16 * 1kB open_sync writes", 1); + printf(_("\nCompare open_sync with different write sizes:\n")); + printf(_("(This is designed to compare the cost of writing 16kB in different write\n" + "open_sync sizes.)\n")); + + test_open_sync(_(" 1 * 16kB open_sync write"), 16); + test_open_sync(_(" 2 * 8kB open_sync writes"), 8); + test_open_sync(_(" 4 * 4kB open_sync writes"), 4); + test_open_sync(_(" 8 * 2kB open_sync writes"), 2); + test_open_sync(_("16 * 1kB open_sync writes"), 1); } /* @@ -422,7 +423,7 @@ test_open_sync(const char *msg, int writes_size) #ifdef OPEN_SYNC_FLAG if ((tmpfile = open(filename, O_RDWR | OPEN_SYNC_FLAG | PG_O_DIRECT, 0)) == -1) - printf(NA_FORMAT, "n/a*\n"); + printf(NA_FORMAT, _("n/a*\n")); else { START_TIMER; @@ -439,7 +440,7 @@ test_open_sync(const char *msg, int writes_size) close(tmpfile); } #else - printf(NA_FORMAT, "n/a\n"); + printf(NA_FORMAT, _("n/a\n")); #endif } @@ -455,9 +456,9 @@ test_file_descriptor_sync(void) * against the same file. Possibly this should be done with writethrough * on platforms which support it. */ - printf("\nTest if fsync on non-write file descriptor is honored:\n"); - printf("(If the times are similar, fsync() can sync data written on a different\n" - "descriptor.)\n"); + printf(_("\nTest if fsync on non-write file descriptor is honored:\n")); + printf(_("(If the times are similar, fsync() can sync data written on a different\n" + "descriptor.)\n")); /* * first write, fsync and close, which is the normal behavior without @@ -521,7 +522,7 @@ test_non_sync(void) /* * Test a simple write without fsync */ - printf("\nNon-sync'ed %dkB writes:\n", XLOG_BLCKSZ_K); + printf(_("\nNon-sync'ed %dkB writes:\n"), XLOG_BLCKSZ_K); printf(LABEL_FORMAT, "write"); fflush(stdout); @@ -598,6 +599,6 @@ process_alarm(LPVOID param) static void die(const char *str) { - fprintf(stderr, "%s: %s\n", str, strerror(errno)); + fprintf(stderr, _("%s: %s\n"), _(str), strerror(errno)); exit(1); } From 59fa9d2d9da46097dd4da5c5f1f07e22a288fccf Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 14 Oct 2016 12:00:00 -0400 Subject: [PATCH 429/871] pg_test_timing: Add NLS Also straighten out use of time unit abbreviations a bit. Reviewed-by: Michael Paquier --- doc/src/sgml/ref/pgtesttiming.sgml | 16 +++++++------- src/bin/pg_test_timing/nls.mk | 4 ++++ src/bin/pg_test_timing/pg_test_timing.c | 28 ++++++++++++++----------- 3 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 src/bin/pg_test_timing/nls.mk diff --git a/doc/src/sgml/ref/pgtesttiming.sgml b/doc/src/sgml/ref/pgtesttiming.sgml index f07a0600ff..e3539cf764 100644 --- a/doc/src/sgml/ref/pgtesttiming.sgml +++ b/doc/src/sgml/ref/pgtesttiming.sgml @@ -96,9 +96,9 @@ Testing timing overhead for 3 seconds. -Per loop time including overhead: 35.96 nsec +Per loop time including overhead: 35.96 ns Histogram of timing durations: -< usec % of total count + < us % of total count 1 96.40465 80435604 2 3.59518 2999652 4 0.00015 126 @@ -109,9 +109,9 @@ Histogram of timing durations: Note that different units are used for the per loop time than the - histogram. The loop can have resolution within a few nanoseconds (nsec), + histogram. The loop can have resolution within a few nanoseconds (ns), while the individual timing calls can only resolve down to one microsecond - (usec). + (us). @@ -157,9 +157,9 @@ EXPLAIN ANALYZE SELECT COUNT(*) FROM t; tsc hpet acpi_pm # echo acpi_pm > /sys/devices/system/clocksource/clocksource0/current_clocksource # pg_test_timing -Per loop time including overhead: 722.92 nsec +Per loop time including overhead: 722.92 ns Histogram of timing durations: -< usec % of total count + < us % of total count 1 27.84870 1155682 2 72.05956 2990371 4 0.07810 3241 @@ -170,7 +170,7 @@ Histogram of timing durations: In this configuration, the sample EXPLAIN ANALYZE above - takes 115.9 ms. That's 1061 nsec of timing overhead, again a small multiple + takes 115.9 ms. That's 1061 ns of timing overhead, again a small multiple of what's measured directly by this utility. That much timing overhead means the actual query itself is only taking a tiny fraction of the accounted for time, most of it is being consumed in overhead instead. In @@ -211,7 +211,7 @@ $ pg_test_timing Testing timing overhead for 3 seconds. Per timing duration including loop overhead: 97.75 ns Histogram of timing durations: -< usec % of total count + < us % of total count 1 90.23734 27694571 2 9.75277 2993204 4 0.00981 3010 diff --git a/src/bin/pg_test_timing/nls.mk b/src/bin/pg_test_timing/nls.mk new file mode 100644 index 0000000000..e12ea5cfdb --- /dev/null +++ b/src/bin/pg_test_timing/nls.mk @@ -0,0 +1,4 @@ +# src/bin/pg_test_timing/nls.mk +CATALOG_NAME = pg_test_timing +AVAIL_LANGUAGES = +GETTEXT_FILES = pg_test_timing.c diff --git a/src/bin/pg_test_timing/pg_test_timing.c b/src/bin/pg_test_timing/pg_test_timing.c index e5c11de6bb..2f1ab7cd60 100644 --- a/src/bin/pg_test_timing/pg_test_timing.c +++ b/src/bin/pg_test_timing/pg_test_timing.c @@ -25,6 +25,7 @@ main(int argc, char *argv[]) { uint64 loop_count; + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_test_timing")); progname = get_progname(argv[0]); handle_args(argc, argv); @@ -51,7 +52,7 @@ handle_args(int argc, char *argv[]) { if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) { - printf("Usage: %s [-d DURATION]\n", progname); + printf(_("Usage: %s [-d DURATION]\n"), progname); exit(0); } if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) @@ -71,7 +72,7 @@ handle_args(int argc, char *argv[]) break; default: - fprintf(stderr, "Try \"%s --help\" for more information.\n", + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); break; @@ -81,23 +82,26 @@ handle_args(int argc, char *argv[]) if (argc > optind) { fprintf(stderr, - "%s: too many command-line arguments (first is \"%s\")\n", + _("%s: too many command-line arguments (first is \"%s\")\n"), progname, argv[optind]); - fprintf(stderr, "Try \"%s --help\" for more information.\n", + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } if (test_duration > 0) { - printf("Testing timing overhead for %d seconds.\n", test_duration); + printf(ngettext("Testing timing overhead for %d second.\n", + "Testing timing overhead for %d seconds.\n", + test_duration), + test_duration); } else { fprintf(stderr, - "%s: duration must be a positive integer (duration is \"%d\")\n", + _("%s: duration must be a positive integer (duration is \"%d\")\n"), progname, test_duration); - fprintf(stderr, "Try \"%s --help\" for more information.\n", + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } @@ -133,8 +137,8 @@ test_timing(int32 duration) /* Did time go backwards? */ if (diff < 0) { - printf("Detected clock going backwards in time.\n"); - printf("Time warp: %d microseconds\n", diff); + fprintf(stderr, _("Detected clock going backwards in time.\n")); + fprintf(stderr, _("Time warp: %d ms\n"), diff); exit(1); } @@ -157,7 +161,7 @@ test_timing(int32 duration) INSTR_TIME_SUBTRACT(end_time, start_time); - printf("Per loop time including overhead: %0.2f nsec\n", + printf(_("Per loop time including overhead: %0.2f ns\n"), INSTR_TIME_GET_DOUBLE(end_time) * 1e9 / loop_count); return loop_count; @@ -173,8 +177,8 @@ output(uint64 loop_count) while (max_bit > 0 && histogram[max_bit] == 0) max_bit--; - printf("Histogram of timing durations:\n"); - printf("%6s %10s %10s\n", "< usec", "% of total", "count"); + printf(_("Histogram of timing durations:\n")); + printf("%6s %10s %10s\n", _("< us"), _("% of total"), _("count")); for (i = 0; i <= max_bit; i++) { From 69d590fffbdcfb50a31a8c78ce87e602002a869f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 15 Oct 2016 12:00:00 -0400 Subject: [PATCH 430/871] pg_xlogdump: Add NLS Reviewed-by: Michael Paquier --- src/bin/pg_xlogdump/nls.mk | 6 +++ src/bin/pg_xlogdump/pg_xlogdump.c | 75 ++++++++++++++++--------------- 2 files changed, 44 insertions(+), 37 deletions(-) create mode 100644 src/bin/pg_xlogdump/nls.mk diff --git a/src/bin/pg_xlogdump/nls.mk b/src/bin/pg_xlogdump/nls.mk new file mode 100644 index 0000000000..2b254c35c6 --- /dev/null +++ b/src/bin/pg_xlogdump/nls.mk @@ -0,0 +1,6 @@ +# src/bin/pg_xlogdump/nls.mk +CATALOG_NAME = pg_xlogdump +AVAIL_LANGUAGES = +GETTEXT_FILES = pg_xlogdump.c +GETTEXT_TRIGGERS = fatal_error +GETTEXT_FLAGS = fatal_error:1:c-format diff --git a/src/bin/pg_xlogdump/pg_xlogdump.c b/src/bin/pg_xlogdump/pg_xlogdump.c index 74903980ae..d070312bef 100644 --- a/src/bin/pg_xlogdump/pg_xlogdump.c +++ b/src/bin/pg_xlogdump/pg_xlogdump.c @@ -79,9 +79,9 @@ fatal_error(const char *fmt,...) fflush(stdout); - fprintf(stderr, "%s: FATAL: ", progname); + fprintf(stderr, _("%s: FATAL: "), progname); va_start(args, fmt); - vfprintf(stderr, fmt, args); + vfprintf(stderr, _(fmt), args); va_end(args); fputc('\n', stderr); @@ -670,27 +670,27 @@ XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats) static void usage(void) { - printf("%s decodes and displays PostgreSQL transaction logs for debugging.\n\n", + printf(_("%s decodes and displays PostgreSQL transaction logs for debugging.\n\n"), progname); - printf("Usage:\n"); - printf(" %s [OPTION]... [STARTSEG [ENDSEG]] \n", progname); - printf("\nOptions:\n"); - printf(" -b, --bkp-details output detailed information about backup blocks\n"); - printf(" -e, --end=RECPTR stop reading at log position RECPTR\n"); - printf(" -f, --follow keep retrying after reaching end of WAL\n"); - printf(" -n, --limit=N number of records to display\n"); - printf(" -p, --path=PATH directory in which to find log segment files\n"); - printf(" (default: ./pg_wal)\n"); - printf(" -r, --rmgr=RMGR only show records generated by resource manager RMGR\n"); - printf(" use --rmgr=list to list valid resource manager names\n"); - printf(" -s, --start=RECPTR start reading at log position RECPTR\n"); - printf(" -t, --timeline=TLI timeline from which to read log records\n"); - printf(" (default: 1 or the value used in STARTSEG)\n"); - printf(" -V, --version output version information, then exit\n"); - printf(" -x, --xid=XID only show records with TransactionId XID\n"); - printf(" -z, --stats[=record] show statistics instead of records\n"); - printf(" (optionally, show per-record statistics)\n"); - printf(" -?, --help show this help, then exit\n"); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]... [STARTSEG [ENDSEG]] \n"), progname); + printf(_("\nOptions:\n")); + printf(_(" -b, --bkp-details output detailed information about backup blocks\n")); + printf(_(" -e, --end=RECPTR stop reading at log position RECPTR\n")); + printf(_(" -f, --follow keep retrying after reaching end of WAL\n")); + printf(_(" -n, --limit=N number of records to display\n")); + printf(_(" -p, --path=PATH directory in which to find log segment files\n" + " (default: ./pg_wal)\n")); + printf(_(" -r, --rmgr=RMGR only show records generated by resource manager RMGR\n" + " use --rmgr=list to list valid resource manager names\n")); + printf(_(" -s, --start=RECPTR start reading at log position RECPTR\n")); + printf(_(" -t, --timeline=TLI timeline from which to read log records\n" + " (default: 1 or the value used in STARTSEG)\n")); + printf(_(" -V, --version output version information, then exit\n")); + printf(_(" -x, --xid=XID only show records with TransactionId XID\n")); + printf(_(" -z, --stats[=record] show statistics instead of records\n" + " (optionally, show per-record statistics)\n")); + printf(_(" -?, --help show this help, then exit\n")); } int @@ -725,6 +725,7 @@ main(int argc, char **argv) int option; int optindex = 0; + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_xlogdump")); progname = get_progname(argv[0]); memset(&private, 0, sizeof(XLogDumpPrivate)); @@ -748,7 +749,7 @@ main(int argc, char **argv) if (argc <= 1) { - fprintf(stderr, "%s: no arguments specified\n", progname); + fprintf(stderr, _("%s: no arguments specified\n"), progname); goto bad_argument; } @@ -763,7 +764,7 @@ main(int argc, char **argv) case 'e': if (sscanf(optarg, "%X/%X", &xlogid, &xrecoff) != 2) { - fprintf(stderr, "%s: could not parse end log position \"%s\"\n", + fprintf(stderr, _("%s: could not parse end log position \"%s\"\n"), progname, optarg); goto bad_argument; } @@ -779,7 +780,7 @@ main(int argc, char **argv) case 'n': if (sscanf(optarg, "%d", &config.stop_after_records) != 1) { - fprintf(stderr, "%s: could not parse limit \"%s\"\n", + fprintf(stderr, _("%s: could not parse limit \"%s\"\n"), progname, optarg); goto bad_argument; } @@ -808,7 +809,7 @@ main(int argc, char **argv) if (config.filter_by_rmgr == -1) { - fprintf(stderr, "%s: resource manager \"%s\" does not exist\n", + fprintf(stderr, _("%s: resource manager \"%s\" does not exist\n"), progname, optarg); goto bad_argument; } @@ -817,7 +818,7 @@ main(int argc, char **argv) case 's': if (sscanf(optarg, "%X/%X", &xlogid, &xrecoff) != 2) { - fprintf(stderr, "%s: could not parse start log position \"%s\"\n", + fprintf(stderr, _("%s: could not parse start log position \"%s\"\n"), progname, optarg); goto bad_argument; } @@ -827,7 +828,7 @@ main(int argc, char **argv) case 't': if (sscanf(optarg, "%d", &private.timeline) != 1) { - fprintf(stderr, "%s: could not parse timeline \"%s\"\n", + fprintf(stderr, _("%s: could not parse timeline \"%s\"\n"), progname, optarg); goto bad_argument; } @@ -839,7 +840,7 @@ main(int argc, char **argv) case 'x': if (sscanf(optarg, "%u", &config.filter_by_xid) != 1) { - fprintf(stderr, "%s: could not parse \"%s\" as a valid xid\n", + fprintf(stderr, _("%s: could not parse \"%s\" as a valid xid\n"), progname, optarg); goto bad_argument; } @@ -854,7 +855,7 @@ main(int argc, char **argv) config.stats_per_record = true; else if (strcmp(optarg, "rmgr") != 0) { - fprintf(stderr, "%s: unrecognised argument to --stats: %s\n", + fprintf(stderr, _("%s: unrecognised argument to --stats: %s\n"), progname, optarg); goto bad_argument; } @@ -868,7 +869,7 @@ main(int argc, char **argv) if ((optind + 2) < argc) { fprintf(stderr, - "%s: too many command-line arguments (first is \"%s\")\n", + _("%s: too many command-line arguments (first is \"%s\")\n"), progname, argv[optind + 2]); goto bad_argument; } @@ -879,7 +880,7 @@ main(int argc, char **argv) if (!verify_directory(private.inpath)) { fprintf(stderr, - "%s: path \"%s\" cannot be opened: %s\n", + _("%s: path \"%s\" cannot be opened: %s\n"), progname, private.inpath, strerror(errno)); goto bad_argument; } @@ -917,7 +918,7 @@ main(int argc, char **argv) else if (!XLByteInSeg(private.startptr, segno)) { fprintf(stderr, - "%s: start log position %X/%X is not inside file \"%s\"\n", + _("%s: start log position %X/%X is not inside file \"%s\"\n"), progname, (uint32) (private.startptr >> 32), (uint32) private.startptr, @@ -961,7 +962,7 @@ main(int argc, char **argv) private.endptr != (segno + 1) * XLogSegSize) { fprintf(stderr, - "%s: end log position %X/%X is not inside file \"%s\"\n", + _("%s: end log position %X/%X is not inside file \"%s\"\n"), progname, (uint32) (private.endptr >> 32), (uint32) private.endptr, @@ -973,7 +974,7 @@ main(int argc, char **argv) /* we don't know what to print */ if (XLogRecPtrIsInvalid(private.startptr)) { - fprintf(stderr, "%s: no start log position given.\n", progname); + fprintf(stderr, _("%s: no start log position given.\n"), progname); goto bad_argument; } @@ -998,7 +999,7 @@ main(int argc, char **argv) * a segment (e.g. we were used in file mode). */ if (first_record != private.startptr && (private.startptr % XLogSegSize) != 0) - printf("first record is after %X/%X, at %X/%X, skipping over %u bytes\n", + printf(_("first record is after %X/%X, at %X/%X, skipping over %u bytes\n"), (uint32) (private.startptr >> 32), (uint32) private.startptr, (uint32) (first_record >> 32), (uint32) first_record, (uint32) (first_record - private.startptr)); @@ -1057,6 +1058,6 @@ main(int argc, char **argv) return EXIT_SUCCESS; bad_argument: - fprintf(stderr, "Try \"%s --help\" for more information.\n", progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); return EXIT_FAILURE; } From 8c48375e5f43ebd832f93c9166d1fe0e639ff806 Mon Sep 17 00:00:00 2001 From: Kevin Grittner Date: Fri, 4 Nov 2016 10:49:50 -0500 Subject: [PATCH 431/871] Implement syntax for transition tables in AFTER triggers. This is infrastructure for the complete SQL standard feature. No support is included at this point for execution nodes or PLs. The intent is to add that soon. As this patch leaves things, standard syntax can create tuplestores to contain old and/or new versions of rows affected by a statement. References to these tuplestores are in the TriggerData structure. C triggers can access the tuplestores directly, so they are usable, but they cannot yet be referenced within a SQL statement. --- doc/src/sgml/catalogs.sgml | 16 ++ doc/src/sgml/ref/create_trigger.sgml | 94 ++++++-- src/backend/commands/tablecmds.c | 5 +- src/backend/commands/trigger.c | 327 +++++++++++++++++++++++++-- src/backend/nodes/copyfuncs.c | 16 ++ src/backend/nodes/equalfuncs.c | 14 ++ src/backend/nodes/outfuncs.c | 13 ++ src/backend/parser/gram.y | 70 +++++- src/backend/utils/adt/ruleutils.c | 23 ++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_trigger.h | 13 +- src/include/commands/trigger.h | 2 + src/include/nodes/nodes.h | 1 + src/include/nodes/parsenodes.h | 17 ++ src/include/parser/kwlist.h | 3 + src/include/utils/reltrigger.h | 7 + 16 files changed, 580 insertions(+), 43 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 29738b07cb..bac169a19e 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -6231,6 +6231,22 @@ representation) for the trigger's WHEN condition, or null if none
+ + + tgoldtable + name + + REFERENCING clause name for OLD TABLE, + or null if none + + + + tgnewtable + name + + REFERENCING clause name for NEW TABLE, + or null if none +
diff --git a/doc/src/sgml/ref/create_trigger.sgml b/doc/src/sgml/ref/create_trigger.sgml index 4bde815012..8590e226e3 100644 --- a/doc/src/sgml/ref/create_trigger.sgml +++ b/doc/src/sgml/ref/create_trigger.sgml @@ -25,6 +25,7 @@ CREATE [ CONSTRAINT ] TRIGGER name ON table_name [ FROM referenced_table_name ] [ NOT DEFERRABLE | [ DEFERRABLE ] [ INITIALLY IMMEDIATE | INITIALLY DEFERRED ] ] + [ REFERENCING { { OLD | NEW } TABLE [ AS ] transition_relation_name } [ ... ] ] [ FOR [ EACH ] { ROW | STATEMENT } ] [ WHEN ( condition ) ] EXECUTE PROCEDURE function_name ( arguments ) @@ -177,6 +178,15 @@ CREATE [ CONSTRAINT ] TRIGGER name when the constraints they implement are violated.
+ + The REFERENCING option is only allowed for an AFTER + trigger which is not a constraint trigger. OLD TABLE may only + be specified once, and only on a trigger which can fire on + UPDATE or DELETE. NEW TABLE may only + be specified once, and only on a trigger which can fire on + UPDATE or INSERT. + + SELECT does not modify any rows so you cannot create SELECT triggers. Rules and views are more @@ -281,6 +291,40 @@ UPDATE OF column_name1 [, column_name2 + + REFERENCING + + + This immediately preceeds the declaration of one or two relations which + can be used to read the before and/or after images of all rows directly + affected by the triggering statement. An AFTER EACH ROW + trigger is allowed to use both these transition relation names and the + row names (OLD and NEW) which reference each + individual row for which the trigger fires. + + + + + + OLD TABLE + NEW TABLE + + + This specifies whether the named relation contains the before or after + images for rows affected by the statement which fired the trigger. + + + + + + transition_relation_name + + + The (unqualified) name to be used within the trigger for this relation. + + + + FOR EACH ROW FOR EACH STATEMENT @@ -474,6 +518,30 @@ CREATE TRIGGER view_insert FOR EACH ROW EXECUTE PROCEDURE view_insert_row(); + + Execute the function check_transfer_balances_to_zero for each + statement to confirm that the transfer rows offset to a net of + zero: + + +CREATE TRIGGER transfer_insert + AFTER INSERT ON transfer + FOR EACH STATEMENT + REFERENCING NEW TABLE AS inserted + EXECUTE PROCEDURE check_transfer_balances_to_zero(); + + + Execute the function check_matching_pairs for each row to + confirm that changes are made to matching pairs at the same time (by the + same statement): + + +CREATE TRIGGER paired_items_update + AFTER UPDATE ON paired_items + FOR EACH ROW + REFERENCING NEW TABLE AS newtab OLD TABLE AS oldtab + EXECUTE PROCEDURE check_matching_pairs(); + @@ -502,24 +570,14 @@ CREATE TRIGGER view_insert - SQL allows you to define aliases for the old - and new rows or tables for use in the definition - of the triggered action (e.g., CREATE TRIGGER ... ON - tablename REFERENCING OLD ROW AS somename NEW ROW AS othername - ...). Since PostgreSQL - allows trigger procedures to be written in any number of - user-defined languages, access to the data is handled in a - language-specific way. - - - - - - PostgreSQL does not allow the old and new - tables to be referenced in statement-level triggers, i.e., the tables - that contain all the old and/or new rows, which are referred to by the - OLD TABLE and NEW TABLE clauses in - the SQL standard. + While transition tables for AFTER triggers are specified + using the REFERENCING clause in the standard way, the row + variables used in FOR EACH ROW triggers may not be + specified in REFERENCING clause. They are available in a + manner which is dependent on the language in which the trigger function + is written. Some languages effectively behave as though there is a + REFERENCING clause containing OLD ROW AS OLD NEW + ROW AS NEW. diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 2137372c23..f97bee5b0e 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -7430,7 +7430,7 @@ validateForeignKeyConstraint(char *conname, trig.tgconstraint = constraintOid; trig.tgdeferrable = FALSE; trig.tginitdeferred = FALSE; - /* we needn't fill in tgargs or tgqual */ + /* we needn't fill in remaining fields */ /* * See if we can do it with a single LEFT JOIN query. A FALSE result @@ -7514,6 +7514,7 @@ CreateFKCheckTrigger(Oid myRelOid, Oid refRelOid, Constraint *fkconstraint, } fk_trigger->columns = NIL; + fk_trigger->transitionRels = NIL; fk_trigger->whenClause = NULL; fk_trigger->isconstraint = true; fk_trigger->deferrable = fkconstraint->deferrable; @@ -7557,6 +7558,7 @@ createForeignKeyTriggers(Relation rel, Oid refRelOid, Constraint *fkconstraint, fk_trigger->timing = TRIGGER_TYPE_AFTER; fk_trigger->events = TRIGGER_TYPE_DELETE; fk_trigger->columns = NIL; + fk_trigger->transitionRels = NIL; fk_trigger->whenClause = NULL; fk_trigger->isconstraint = true; fk_trigger->constrrel = NULL; @@ -7611,6 +7613,7 @@ createForeignKeyTriggers(Relation rel, Oid refRelOid, Constraint *fkconstraint, fk_trigger->timing = TRIGGER_TYPE_AFTER; fk_trigger->events = TRIGGER_TYPE_UPDATE; fk_trigger->columns = NIL; + fk_trigger->transitionRels = NIL; fk_trigger->whenClause = NULL; fk_trigger->isconstraint = true; fk_trigger->constrrel = NULL; diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 9de22a13d7..1c264b7736 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -164,6 +164,8 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString, Oid constrrelid = InvalidOid; ObjectAddress myself, referenced; + char *oldtablename = NULL; + char *newtablename = NULL; if (OidIsValid(relOid)) rel = heap_open(relOid, ShareRowExclusiveLock); @@ -309,6 +311,87 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString, errmsg("INSTEAD OF triggers cannot have column lists"))); } + /* + * We don't yet support naming ROW transition variables, but the parser + * recognizes the syntax so we can give a nicer message here. + * + * Per standard, REFERENCING TABLE names are only allowed on AFTER + * triggers. Per standard, REFERENCING ROW names are not allowed with FOR + * EACH STATEMENT. Per standard, each OLD/NEW, ROW/TABLE permutation is + * only allowed once. Per standard, OLD may not be specified when + * creating a trigger only for INSERT, and NEW may not be specified when + * creating a trigger only for DELETE. + * + * Notice that the standard allows an AFTER ... FOR EACH ROW trigger to + * reference both ROW and TABLE transition data. + */ + if (stmt->transitionRels != NIL) + { + List *varList = stmt->transitionRels; + ListCell *lc; + + foreach(lc, varList) + { + TriggerTransition *tt = (TriggerTransition *) lfirst(lc); + + Assert(IsA(tt, TriggerTransition)); + + if (!(tt->isTable)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ROW variable naming in the REFERENCING clause is not supported"), + errhint("Use OLD TABLE or NEW TABLE for naming transition tables."))); + + /* + * Because of the above test, we omit further ROW-related testing + * below. If we later allow naming OLD and NEW ROW variables, + * adjustments will be needed below. + */ + + if (stmt->timing != TRIGGER_TYPE_AFTER) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("transition table name can only be specified for an AFTER trigger"))); + + if (tt->isNew) + { + if (!(TRIGGER_FOR_INSERT(tgtype) || + TRIGGER_FOR_UPDATE(tgtype))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("NEW TABLE can only be specified for an INSERT or UPDATE trigger"))); + + if (newtablename != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("NEW TABLE cannot be specified multiple times"))); + + newtablename = tt->name; + } + else + { + if (!(TRIGGER_FOR_DELETE(tgtype) || + TRIGGER_FOR_UPDATE(tgtype))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("OLD TABLE can only be specified for a DELETE or UPDATE trigger"))); + + if (oldtablename != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("OLD TABLE cannot be specified multiple times"))); + + oldtablename = tt->name; + } + } + + if (newtablename != NULL && oldtablename != NULL && + strcmp(newtablename, oldtablename) == 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("OLD TABLE name and NEW TABLE name cannot be the same"))); + } + /* * Parse the WHEN clause, if any */ @@ -664,6 +747,17 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString, else nulls[Anum_pg_trigger_tgqual - 1] = true; + if (oldtablename) + values[Anum_pg_trigger_tgoldtable - 1] = DirectFunctionCall1(namein, + CStringGetDatum(oldtablename)); + else + nulls[Anum_pg_trigger_tgoldtable - 1] = true; + if (newtablename) + values[Anum_pg_trigger_tgnewtable - 1] = DirectFunctionCall1(namein, + CStringGetDatum(newtablename)); + else + nulls[Anum_pg_trigger_tgnewtable - 1] = true; + tuple = heap_form_tuple(tgrel->rd_att, values, nulls); /* force tuple to have the desired OID */ @@ -682,6 +776,10 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString, pfree(DatumGetPointer(values[Anum_pg_trigger_tgname - 1])); pfree(DatumGetPointer(values[Anum_pg_trigger_tgargs - 1])); pfree(DatumGetPointer(values[Anum_pg_trigger_tgattr - 1])); + if (oldtablename) + pfree(DatumGetPointer(values[Anum_pg_trigger_tgoldtable - 1])); + if (newtablename) + pfree(DatumGetPointer(values[Anum_pg_trigger_tgnewtable - 1])); /* * Update relation's pg_class entry. Crucial side-effect: other backends @@ -1584,6 +1682,23 @@ RelationBuildTriggers(Relation relation) } else build->tgargs = NULL; + + datum = fastgetattr(htup, Anum_pg_trigger_tgoldtable, + tgrel->rd_att, &isnull); + if (!isnull) + build->tgoldtable = + DatumGetCString(DirectFunctionCall1(nameout, datum)); + else + build->tgoldtable = NULL; + + datum = fastgetattr(htup, Anum_pg_trigger_tgnewtable, + tgrel->rd_att, &isnull); + if (!isnull) + build->tgnewtable = + DatumGetCString(DirectFunctionCall1(nameout, datum)); + else + build->tgnewtable = NULL; + datum = fastgetattr(htup, Anum_pg_trigger_tgqual, tgrel->rd_att, &isnull); if (!isnull) @@ -1680,6 +1795,19 @@ SetTriggerFlags(TriggerDesc *trigdesc, Trigger *trigger) trigdesc->trig_truncate_after_statement |= TRIGGER_TYPE_MATCHES(tgtype, TRIGGER_TYPE_STATEMENT, TRIGGER_TYPE_AFTER, TRIGGER_TYPE_TRUNCATE); + + trigdesc->trig_insert_new_table |= + (TRIGGER_FOR_INSERT(tgtype) && + TRIGGER_USES_TRANSITION_TABLE(trigger->tgnewtable)); + trigdesc->trig_update_old_table |= + (TRIGGER_FOR_UPDATE(tgtype) && + TRIGGER_USES_TRANSITION_TABLE(trigger->tgoldtable)); + trigdesc->trig_update_new_table |= + (TRIGGER_FOR_UPDATE(tgtype) && + TRIGGER_USES_TRANSITION_TABLE(trigger->tgnewtable)); + trigdesc->trig_delete_old_table |= + (TRIGGER_FOR_DELETE(tgtype) && + TRIGGER_USES_TRANSITION_TABLE(trigger->tgoldtable)); } /* @@ -1729,6 +1857,10 @@ CopyTriggerDesc(TriggerDesc *trigdesc) } if (trigger->tgqual) trigger->tgqual = pstrdup(trigger->tgqual); + if (trigger->tgoldtable) + trigger->tgoldtable = pstrdup(trigger->tgoldtable); + if (trigger->tgnewtable) + trigger->tgnewtable = pstrdup(trigger->tgnewtable); trigger++; } @@ -1761,6 +1893,10 @@ FreeTriggerDesc(TriggerDesc *trigdesc) } if (trigger->tgqual) pfree(trigger->tgqual); + if (trigger->tgoldtable) + pfree(trigger->tgoldtable); + if (trigger->tgnewtable) + pfree(trigger->tgnewtable); trigger++; } pfree(trigdesc->triggers); @@ -1839,6 +1975,18 @@ equalTriggerDescs(TriggerDesc *trigdesc1, TriggerDesc *trigdesc2) return false; else if (strcmp(trig1->tgqual, trig2->tgqual) != 0) return false; + if (trig1->tgoldtable == NULL && trig2->tgoldtable == NULL) + /* ok */ ; + else if (trig1->tgoldtable == NULL || trig2->tgoldtable == NULL) + return false; + else if (strcmp(trig1->tgoldtable, trig2->tgoldtable) != 0) + return false; + if (trig1->tgnewtable == NULL && trig2->tgnewtable == NULL) + /* ok */ ; + else if (trig1->tgnewtable == NULL || trig2->tgnewtable == NULL) + return false; + else if (strcmp(trig1->tgnewtable, trig2->tgnewtable) != 0) + return false; } } else if (trigdesc2 != NULL) @@ -1870,6 +2018,18 @@ ExecCallTriggerFunc(TriggerData *trigdata, Datum result; MemoryContext oldContext; + /* + * Protect against code paths that may fail to initialize transition table + * info. + */ + Assert(((TRIGGER_FIRED_BY_INSERT(trigdata->tg_event) || + TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event) || + TRIGGER_FIRED_BY_DELETE(trigdata->tg_event)) && + TRIGGER_FIRED_AFTER(trigdata->tg_event) && + !(trigdata->tg_event & AFTER_TRIGGER_DEFERRABLE) && + !(trigdata->tg_event & AFTER_TRIGGER_INITDEFERRED)) || + (trigdata->tg_oldtable == NULL && trigdata->tg_newtable == NULL)); + finfo += tgindx; /* @@ -1960,6 +2120,8 @@ ExecBSInsertTriggers(EState *estate, ResultRelInfo *relinfo) LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_trigtuple = NULL; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_trigtuplebuf = InvalidBuffer; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) @@ -2017,6 +2179,8 @@ ExecBRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TRIGGER_EVENT_BEFORE; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2070,7 +2234,8 @@ ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, { TriggerDesc *trigdesc = relinfo->ri_TrigDesc; - if (trigdesc && trigdesc->trig_insert_after_row) + if (trigdesc && + (trigdesc->trig_insert_after_row || trigdesc->trig_insert_new_table)) AfterTriggerSaveEvent(estate, relinfo, TRIGGER_EVENT_INSERT, true, NULL, trigtuple, recheckIndexes, NULL); } @@ -2092,6 +2257,8 @@ ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TRIGGER_EVENT_INSTEAD; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2159,6 +2326,8 @@ ExecBSDeleteTriggers(EState *estate, ResultRelInfo *relinfo) LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_trigtuple = NULL; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_trigtuplebuf = InvalidBuffer; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) @@ -2230,6 +2399,8 @@ ExecBRDeleteTriggers(EState *estate, EPQState *epqstate, TRIGGER_EVENT_BEFORE; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2273,7 +2444,8 @@ ExecARDeleteTriggers(EState *estate, ResultRelInfo *relinfo, { TriggerDesc *trigdesc = relinfo->ri_TrigDesc; - if (trigdesc && trigdesc->trig_delete_after_row) + if (trigdesc && + (trigdesc->trig_delete_after_row || trigdesc->trig_delete_old_table)) { HeapTuple trigtuple; @@ -2310,6 +2482,8 @@ ExecIRDeleteTriggers(EState *estate, ResultRelInfo *relinfo, TRIGGER_EVENT_INSTEAD; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2363,6 +2537,8 @@ ExecBSUpdateTriggers(EState *estate, ResultRelInfo *relinfo) LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_trigtuple = NULL; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_trigtuplebuf = InvalidBuffer; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) @@ -2464,6 +2640,8 @@ ExecBRUpdateTriggers(EState *estate, EPQState *epqstate, TRIGGER_EVENT_ROW | TRIGGER_EVENT_BEFORE; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; updatedCols = GetUpdatedColumns(relinfo, estate); for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2528,7 +2706,8 @@ ExecARUpdateTriggers(EState *estate, ResultRelInfo *relinfo, { TriggerDesc *trigdesc = relinfo->ri_TrigDesc; - if (trigdesc && trigdesc->trig_update_after_row) + if (trigdesc && (trigdesc->trig_update_after_row || + trigdesc->trig_update_old_table || trigdesc->trig_update_new_table)) { HeapTuple trigtuple; @@ -2567,6 +2746,8 @@ ExecIRUpdateTriggers(EState *estate, ResultRelInfo *relinfo, TRIGGER_EVENT_ROW | TRIGGER_EVENT_INSTEAD; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; for (i = 0; i < trigdesc->numtriggers; i++) { Trigger *trigger = &trigdesc->triggers[i]; @@ -2635,6 +2816,8 @@ ExecBSTruncateTriggers(EState *estate, ResultRelInfo *relinfo) LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_trigtuple = NULL; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_trigtuplebuf = InvalidBuffer; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) @@ -3163,8 +3346,11 @@ typedef struct AfterTriggerEventList * fdw_tuplestores[query_depth] is a tuplestore containing the foreign tuples * needed for the current query. * - * maxquerydepth is just the allocated length of query_stack and - * fdw_tuplestores. + * old_tuplestores[query_depth] and new_tuplestores[query_depth] hold the + * transition relations for the current query. + * + * maxquerydepth is just the allocated length of query_stack and the + * tuplestores. * * state_stack is a stack of pointers to saved copies of the SET CONSTRAINTS * state data; each subtransaction level that modifies that state first @@ -3193,7 +3379,9 @@ typedef struct AfterTriggersData AfterTriggerEventList events; /* deferred-event list */ int query_depth; /* current query list index */ AfterTriggerEventList *query_stack; /* events pending from each query */ - Tuplestorestate **fdw_tuplestores; /* foreign tuples from each query */ + Tuplestorestate **fdw_tuplestores; /* foreign tuples for one row from each query */ + Tuplestorestate **old_tuplestores; /* all old tuples from each query */ + Tuplestorestate **new_tuplestores; /* all new tuples from each query */ int maxquerydepth; /* allocated len of above array */ MemoryContext event_cxt; /* memory context for events, if any */ @@ -3222,14 +3410,16 @@ static SetConstraintState SetConstraintStateAddItem(SetConstraintState state, /* - * Gets the current query fdw tuplestore and initializes it if necessary + * Gets a current query transition tuplestore and initializes it if necessary. + * This can be holding a single transition row tuple (in the case of an FDW) + * or a transition table (for an AFTER trigger). */ static Tuplestorestate * -GetCurrentFDWTuplestore(void) +GetTriggerTransitionTuplestore(Tuplestorestate **tss) { Tuplestorestate *ret; - ret = afterTriggers.fdw_tuplestores[afterTriggers.query_depth]; + ret = tss[afterTriggers.query_depth]; if (ret == NULL) { MemoryContext oldcxt; @@ -3256,7 +3446,7 @@ GetCurrentFDWTuplestore(void) CurrentResourceOwner = saveResourceOwner; MemoryContextSwitchTo(oldcxt); - afterTriggers.fdw_tuplestores[afterTriggers.query_depth] = ret; + tss[afterTriggers.query_depth] = ret; } return ret; @@ -3554,7 +3744,9 @@ AfterTriggerExecute(AfterTriggerEvent event, { case AFTER_TRIGGER_FDW_FETCH: { - Tuplestorestate *fdw_tuplestore = GetCurrentFDWTuplestore(); + Tuplestorestate *fdw_tuplestore = + GetTriggerTransitionTuplestore + (afterTriggers.fdw_tuplestores); if (!tuplestore_gettupleslot(fdw_tuplestore, true, false, trig_tuple_slot1)) @@ -3623,6 +3815,20 @@ AfterTriggerExecute(AfterTriggerEvent event, } } + /* + * Set up the tuplestore information. + */ + if (LocTriggerData.tg_trigger->tgoldtable) + LocTriggerData.tg_oldtable = + GetTriggerTransitionTuplestore(afterTriggers.old_tuplestores); + else + LocTriggerData.tg_oldtable = NULL; + if (LocTriggerData.tg_trigger->tgnewtable) + LocTriggerData.tg_newtable = + GetTriggerTransitionTuplestore(afterTriggers.new_tuplestores); + else + LocTriggerData.tg_newtable = NULL; + /* * Setup the remaining trigger information */ @@ -3912,6 +4118,8 @@ AfterTriggerBeginXact(void) Assert(afterTriggers.state == NULL); Assert(afterTriggers.query_stack == NULL); Assert(afterTriggers.fdw_tuplestores == NULL); + Assert(afterTriggers.old_tuplestores == NULL); + Assert(afterTriggers.new_tuplestores == NULL); Assert(afterTriggers.maxquerydepth == 0); Assert(afterTriggers.event_cxt == NULL); Assert(afterTriggers.events.head == NULL); @@ -3956,6 +4164,8 @@ AfterTriggerEndQuery(EState *estate) { AfterTriggerEventList *events; Tuplestorestate *fdw_tuplestore; + Tuplestorestate *old_tuplestore; + Tuplestorestate *new_tuplestore; /* Must be inside a query, too */ Assert(afterTriggers.query_depth >= 0); @@ -4014,6 +4224,18 @@ AfterTriggerEndQuery(EState *estate) tuplestore_end(fdw_tuplestore); afterTriggers.fdw_tuplestores[afterTriggers.query_depth] = NULL; } + old_tuplestore = afterTriggers.old_tuplestores[afterTriggers.query_depth]; + if (old_tuplestore) + { + tuplestore_end(old_tuplestore); + afterTriggers.old_tuplestores[afterTriggers.query_depth] = NULL; + } + new_tuplestore = afterTriggers.new_tuplestores[afterTriggers.query_depth]; + if (new_tuplestore) + { + tuplestore_end(new_tuplestore); + afterTriggers.new_tuplestores[afterTriggers.query_depth] = NULL; + } afterTriggerFreeEventList(&afterTriggers.query_stack[afterTriggers.query_depth]); afterTriggers.query_depth--; @@ -4127,6 +4349,8 @@ AfterTriggerEndXact(bool isCommit) */ afterTriggers.query_stack = NULL; afterTriggers.fdw_tuplestores = NULL; + afterTriggers.old_tuplestores = NULL; + afterTriggers.new_tuplestores = NULL; afterTriggers.maxquerydepth = 0; afterTriggers.state = NULL; @@ -4259,6 +4483,18 @@ AfterTriggerEndSubXact(bool isCommit) tuplestore_end(ts); afterTriggers.fdw_tuplestores[afterTriggers.query_depth] = NULL; } + ts = afterTriggers.old_tuplestores[afterTriggers.query_depth]; + if (ts) + { + tuplestore_end(ts); + afterTriggers.old_tuplestores[afterTriggers.query_depth] = NULL; + } + ts = afterTriggers.new_tuplestores[afterTriggers.query_depth]; + if (ts) + { + tuplestore_end(ts); + afterTriggers.new_tuplestores[afterTriggers.query_depth] = NULL; + } afterTriggerFreeEventList(&afterTriggers.query_stack[afterTriggers.query_depth]); } @@ -4338,6 +4574,12 @@ AfterTriggerEnlargeQueryState(void) afterTriggers.fdw_tuplestores = (Tuplestorestate **) MemoryContextAllocZero(TopTransactionContext, new_alloc * sizeof(Tuplestorestate *)); + afterTriggers.old_tuplestores = (Tuplestorestate **) + MemoryContextAllocZero(TopTransactionContext, + new_alloc * sizeof(Tuplestorestate *)); + afterTriggers.new_tuplestores = (Tuplestorestate **) + MemoryContextAllocZero(TopTransactionContext, + new_alloc * sizeof(Tuplestorestate *)); afterTriggers.maxquerydepth = new_alloc; } else @@ -4353,9 +4595,19 @@ AfterTriggerEnlargeQueryState(void) afterTriggers.fdw_tuplestores = (Tuplestorestate **) repalloc(afterTriggers.fdw_tuplestores, new_alloc * sizeof(Tuplestorestate *)); + afterTriggers.old_tuplestores = (Tuplestorestate **) + repalloc(afterTriggers.old_tuplestores, + new_alloc * sizeof(Tuplestorestate *)); + afterTriggers.new_tuplestores = (Tuplestorestate **) + repalloc(afterTriggers.new_tuplestores, + new_alloc * sizeof(Tuplestorestate *)); /* Clear newly-allocated slots for subsequent lazy initialization. */ memset(afterTriggers.fdw_tuplestores + old_alloc, 0, (new_alloc - old_alloc) * sizeof(Tuplestorestate *)); + memset(afterTriggers.old_tuplestores + old_alloc, + 0, (new_alloc - old_alloc) * sizeof(Tuplestorestate *)); + memset(afterTriggers.new_tuplestores + old_alloc, + 0, (new_alloc - old_alloc) * sizeof(Tuplestorestate *)); afterTriggers.maxquerydepth = new_alloc; } @@ -4800,7 +5052,14 @@ AfterTriggerPendingOnRel(Oid relid) * * NOTE: this is called whenever there are any triggers associated with * the event (even if they are disabled). This function decides which - * triggers actually need to be queued. + * triggers actually need to be queued. It is also called after each row, + * even if there are no triggers for that event, if there are any AFTER + * STATEMENT triggers for the statement which use transition tables, so that + * the transition tuplestores can be built. + * + * Transition tuplestores are built now, rather than when events are pulled + * off of the queue because AFTER ROW triggers are allowed to select from the + * transition tables for the statement. * ---------- */ static void @@ -4831,6 +5090,46 @@ AfterTriggerSaveEvent(EState *estate, ResultRelInfo *relinfo, if (afterTriggers.query_depth >= afterTriggers.maxquerydepth) AfterTriggerEnlargeQueryState(); + /* + * If the relation has AFTER ... FOR EACH ROW triggers, capture rows into + * transition tuplestores for this depth. + */ + if (row_trigger) + { + if ((event == TRIGGER_EVENT_DELETE && + trigdesc->trig_delete_old_table) || + (event == TRIGGER_EVENT_UPDATE && + trigdesc->trig_update_old_table)) + { + Tuplestorestate *old_tuplestore; + + Assert(oldtup != NULL); + old_tuplestore = + GetTriggerTransitionTuplestore + (afterTriggers.old_tuplestores); + tuplestore_puttuple(old_tuplestore, oldtup); + } + if ((event == TRIGGER_EVENT_INSERT && + trigdesc->trig_insert_new_table) || + (event == TRIGGER_EVENT_UPDATE && + trigdesc->trig_update_new_table)) + { + Tuplestorestate *new_tuplestore; + + Assert(newtup != NULL); + new_tuplestore = + GetTriggerTransitionTuplestore + (afterTriggers.new_tuplestores); + tuplestore_puttuple(new_tuplestore, newtup); + } + + /* If transition tables are the only reason we're here, return. */ + if ((event == TRIGGER_EVENT_DELETE && !trigdesc->trig_delete_after_row) || + (event == TRIGGER_EVENT_INSERT && !trigdesc->trig_insert_after_row) || + (event == TRIGGER_EVENT_UPDATE && !trigdesc->trig_update_after_row)) + return; + } + /* * Validate the event code and collect the associated tuple CTIDs. * @@ -4928,7 +5227,9 @@ AfterTriggerSaveEvent(EState *estate, ResultRelInfo *relinfo, { if (fdw_tuplestore == NULL) { - fdw_tuplestore = GetCurrentFDWTuplestore(); + fdw_tuplestore = + GetTriggerTransitionTuplestore + (afterTriggers.fdw_tuplestores); new_event.ate_flags = AFTER_TRIGGER_FDW_FETCH; } else diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 71714bc1d6..04e49b7795 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2718,6 +2718,18 @@ _copyRoleSpec(const RoleSpec *from) return newnode; } +static TriggerTransition * +_copyTriggerTransition(const TriggerTransition *from) +{ + TriggerTransition *newnode = makeNode(TriggerTransition); + + COPY_STRING_FIELD(name); + COPY_SCALAR_FIELD(isNew); + COPY_SCALAR_FIELD(isTable); + + return newnode; +} + static Query * _copyQuery(const Query *from) { @@ -3893,6 +3905,7 @@ _copyCreateTrigStmt(const CreateTrigStmt *from) COPY_NODE_FIELD(columns); COPY_NODE_FIELD(whenClause); COPY_SCALAR_FIELD(isconstraint); + COPY_NODE_FIELD(transitionRels); COPY_SCALAR_FIELD(deferrable); COPY_SCALAR_FIELD(initdeferred); COPY_NODE_FIELD(constrrel); @@ -5088,6 +5101,9 @@ copyObject(const void *from) case T_RoleSpec: retval = _copyRoleSpec(from); break; + case T_TriggerTransition: + retval = _copyTriggerTransition(from); + break; /* * MISCELLANEOUS NODES diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 29a090fc48..2eaf41c37f 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1905,6 +1905,7 @@ _equalCreateTrigStmt(const CreateTrigStmt *a, const CreateTrigStmt *b) COMPARE_NODE_FIELD(columns); COMPARE_NODE_FIELD(whenClause); COMPARE_SCALAR_FIELD(isconstraint); + COMPARE_NODE_FIELD(transitionRels); COMPARE_SCALAR_FIELD(deferrable); COMPARE_SCALAR_FIELD(initdeferred); COMPARE_NODE_FIELD(constrrel); @@ -2634,6 +2635,16 @@ _equalRoleSpec(const RoleSpec *a, const RoleSpec *b) return true; } +static bool +_equalTriggerTransition(const TriggerTransition *a, const TriggerTransition *b) +{ + COMPARE_STRING_FIELD(name); + COMPARE_SCALAR_FIELD(isNew); + COMPARE_SCALAR_FIELD(isTable); + + return true; +} + /* * Stuff from pg_list.h */ @@ -3387,6 +3398,9 @@ equal(const void *a, const void *b) case T_RoleSpec: retval = _equalRoleSpec(a, b); break; + case T_TriggerTransition: + retval = _equalTriggerTransition(a, b); + break; default: elog(ERROR, "unrecognized node type: %d", diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index ae869547f3..748b687929 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2561,6 +2561,16 @@ _outXmlSerialize(StringInfo str, const XmlSerialize *node) WRITE_LOCATION_FIELD(location); } +static void +_outTriggerTransition(StringInfo str, const TriggerTransition *node) +{ + WRITE_NODE_TYPE("TRIGGERTRANSITION"); + + WRITE_STRING_FIELD(name); + WRITE_BOOL_FIELD(isNew); + WRITE_BOOL_FIELD(isTable); +} + static void _outColumnDef(StringInfo str, const ColumnDef *node) { @@ -3852,6 +3862,9 @@ outNode(StringInfo str, const void *obj) case T_ForeignKeyCacheInfo: _outForeignKeyCacheInfo(str, obj); break; + case T_TriggerTransition: + _outTriggerTransition(str, obj); + break; default: diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 5547fc8658..0ec1cd345b 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -310,6 +310,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type TriggerEvents TriggerOneEvent %type TriggerFuncArg %type TriggerWhen +%type TransitionRelName +%type TransitionRowOrTable TransitionOldOrNew +%type TriggerTransition %type event_trigger_when_list event_trigger_value_list %type event_trigger_when_item @@ -374,6 +377,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); create_generic_options alter_generic_options relation_expr_list dostmt_opt_list transform_element_list transform_type_list + TriggerTransitions TriggerReferencing %type group_by_list %type group_by_item empty_grouping_set rollup_clause cube_clause @@ -610,11 +614,11 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); MAPPING MATCH MATERIALIZED MAXVALUE METHOD MINUTE_P MINVALUE MODE MONTH_P MOVE - NAME_P NAMES NATIONAL NATURAL NCHAR NEXT NO NONE + NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NO NONE NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF NULLS_P NUMERIC - OBJECT_P OF OFF OFFSET OIDS ON ONLY OPERATOR OPTION OPTIONS OR + OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OPTIONS OR ORDER ORDINALITY OUT_P OUTER_P OVER OVERLAPS OVERLAY OWNED OWNER PARALLEL PARSER PARTIAL PARTITION PASSING PASSWORD PLACING PLANS POLICY @@ -623,8 +627,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); QUOTE - RANGE READ REAL REASSIGN RECHECK RECURSIVE REF REFERENCES REFRESH REINDEX - RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA + RANGE READ REAL REASSIGN RECHECK RECURSIVE REF REFERENCES REFERENCING + REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA RESET RESTART RESTRICT RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP ROW ROWS RULE @@ -4748,19 +4752,20 @@ CreateAmStmt: CREATE ACCESS METHOD name TYPE_P INDEX HANDLER handler_name CreateTrigStmt: CREATE TRIGGER name TriggerActionTime TriggerEvents ON - qualified_name TriggerForSpec TriggerWhen + qualified_name TriggerReferencing TriggerForSpec TriggerWhen EXECUTE PROCEDURE func_name '(' TriggerFuncArgs ')' { CreateTrigStmt *n = makeNode(CreateTrigStmt); n->trigname = $3; n->relation = $7; - n->funcname = $12; - n->args = $14; - n->row = $8; + n->funcname = $13; + n->args = $15; + n->row = $9; n->timing = $4; n->events = intVal(linitial($5)); n->columns = (List *) lsecond($5); - n->whenClause = $9; + n->whenClause = $10; + n->transitionRels = $8; n->isconstraint = FALSE; n->deferrable = FALSE; n->initdeferred = FALSE; @@ -4782,6 +4787,7 @@ CreateTrigStmt: n->events = intVal(linitial($6)); n->columns = (List *) lsecond($6); n->whenClause = $14; + n->transitionRels = NIL; n->isconstraint = TRUE; processCASbits($10, @10, "TRIGGER", &n->deferrable, &n->initdeferred, NULL, @@ -4834,6 +4840,49 @@ TriggerOneEvent: { $$ = list_make2(makeInteger(TRIGGER_TYPE_TRUNCATE), NIL); } ; +TriggerReferencing: + REFERENCING TriggerTransitions { $$ = $2; } + | /*EMPTY*/ { $$ = NIL; } + ; + +TriggerTransitions: + TriggerTransition { $$ = list_make1($1); } + | TriggerTransitions TriggerTransition { $$ = lappend($1, $2); } + ; + +TriggerTransition: + TransitionOldOrNew TransitionRowOrTable opt_as TransitionRelName + { + TriggerTransition *n = makeNode(TriggerTransition); + n->name = $4; + n->isNew = $1; + n->isTable = $2; + $$ = (Node *)n; + } + ; + +TransitionOldOrNew: + NEW { $$ = TRUE; } + | OLD { $$ = FALSE; } + ; + +TransitionRowOrTable: + TABLE { $$ = TRUE; } + /* + * According to the standard, lack of a keyword here implies ROW. + * Support for that would require prohibiting ROW entirely here, + * reserving the keyword ROW, and/or requiring AS (instead of + * allowing it to be optional, as the standard specifies) as the + * next token. Requiring ROW seems cleanest and easiest to + * explain. + */ + | ROW { $$ = FALSE; } + ; + +TransitionRelName: + ColId { $$ = $1; } + ; + TriggerForSpec: FOR TriggerForOptEach TriggerForType { @@ -13810,6 +13859,7 @@ unreserved_keyword: | MOVE | NAME_P | NAMES + | NEW | NEXT | NO | NOTHING @@ -13820,6 +13870,7 @@ unreserved_keyword: | OF | OFF | OIDS + | OLD | OPERATOR | OPTION | OPTIONS @@ -13851,6 +13902,7 @@ unreserved_keyword: | RECHECK | RECURSIVE | REF + | REFERENCING | REFRESH | REINDEX | RELATIVE_P diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 8a81d7a078..a3a4174abf 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -813,6 +813,8 @@ pg_get_triggerdef_worker(Oid trigid, bool pretty) SysScanDesc tgscan; int findx = 0; char *tgname; + char *tgoldtable; + char *tgnewtable; Oid argtypes[1]; /* dummy */ Datum value; bool isnull; @@ -924,6 +926,27 @@ pg_get_triggerdef_worker(Oid trigid, bool pretty) appendStringInfoString(&buf, "IMMEDIATE "); } + value = fastgetattr(ht_trig, Anum_pg_trigger_tgoldtable, + tgrel->rd_att, &isnull); + if (!isnull) + tgoldtable = NameStr(*((NameData *) DatumGetPointer(value))); + else + tgoldtable = NULL; + value = fastgetattr(ht_trig, Anum_pg_trigger_tgnewtable, + tgrel->rd_att, &isnull); + if (!isnull) + tgnewtable = NameStr(*((NameData *) DatumGetPointer(value))); + else + tgnewtable = NULL; + if (tgoldtable != NULL || tgnewtable != NULL) + { + appendStringInfoString(&buf, "REFERENCING "); + if (tgoldtable != NULL) + appendStringInfo(&buf, "OLD TABLE AS %s ", tgoldtable); + if (tgnewtable != NULL) + appendStringInfo(&buf, "NEW TABLE AS %s ", tgnewtable); + } + if (TRIGGER_FOR_ROW(trigrec->tgtype)) appendStringInfoString(&buf, "FOR EACH ROW "); else diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index cd3048db86..880559650a 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201610201 +#define CATALOG_VERSION_NO 201611041 #endif diff --git a/src/include/catalog/pg_trigger.h b/src/include/catalog/pg_trigger.h index eb39c50e63..da6a7f3a2e 100644 --- a/src/include/catalog/pg_trigger.h +++ b/src/include/catalog/pg_trigger.h @@ -59,6 +59,8 @@ CATALOG(pg_trigger,2620) #ifdef CATALOG_VARLEN bytea tgargs BKI_FORCE_NOT_NULL; /* first\000second\000tgnargs\000 */ pg_node_tree tgqual; /* WHEN expression, or NULL if none */ + NameData tgoldtable; /* old transition table, or NULL if none */ + NameData tgnewtable; /* new transition table, or NULL if none */ #endif } FormData_pg_trigger; @@ -73,7 +75,7 @@ typedef FormData_pg_trigger *Form_pg_trigger; * compiler constants for pg_trigger * ---------------- */ -#define Natts_pg_trigger 15 +#define Natts_pg_trigger 17 #define Anum_pg_trigger_tgrelid 1 #define Anum_pg_trigger_tgname 2 #define Anum_pg_trigger_tgfoid 3 @@ -89,6 +91,8 @@ typedef FormData_pg_trigger *Form_pg_trigger; #define Anum_pg_trigger_tgattr 13 #define Anum_pg_trigger_tgargs 14 #define Anum_pg_trigger_tgqual 15 +#define Anum_pg_trigger_tgoldtable 16 +#define Anum_pg_trigger_tgnewtable 17 /* Bits within tgtype */ #define TRIGGER_TYPE_ROW (1 << 0) @@ -142,4 +146,11 @@ typedef FormData_pg_trigger *Form_pg_trigger; #define TRIGGER_TYPE_MATCHES(type, level, timing, event) \ (((type) & (TRIGGER_TYPE_LEVEL_MASK | TRIGGER_TYPE_TIMING_MASK | (event))) == ((level) | (timing) | (event))) +/* + * Macro to determine whether tgnewtable or tgoldtable has been specified for + * a trigger. + */ +#define TRIGGER_USES_TRANSITION_TABLE(namepointer) \ + ((namepointer) != (char *) NULL) + #endif /* PG_TRIGGER_H */ diff --git a/src/include/commands/trigger.h b/src/include/commands/trigger.h index 0ed7c86eb2..c6e3e2c234 100644 --- a/src/include/commands/trigger.h +++ b/src/include/commands/trigger.h @@ -37,6 +37,8 @@ typedef struct TriggerData Trigger *tg_trigger; Buffer tg_trigtuplebuf; Buffer tg_newtuplebuf; + Tuplestorestate *tg_oldtable; + Tuplestorestate *tg_newtable; } TriggerData; /* diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 88297bbe80..cb9307cd00 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -453,6 +453,7 @@ typedef enum NodeTag T_OnConflictClause, T_CommonTableExpr, T_RoleSpec, + T_TriggerTransition, /* * TAGS FOR REPLICATION GRAMMAR PARSE NODES (replnodes.h) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 6de2cab6b2..9b600a5f76 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1204,6 +1204,21 @@ typedef struct CommonTableExpr ((Query *) (cte)->ctequery)->targetList : \ ((Query *) (cte)->ctequery)->returningList) +/* + * TriggerTransition - + * representation of transition row or table naming clause + * + * Only transition tables are initially supported in the syntax, and only for + * AFTER triggers, but other permutations are accepted by the parser so we can + * give a meaningful message from C code. + */ +typedef struct TriggerTransition +{ + NodeTag type; + char *name; + bool isNew; + bool isTable; +} TriggerTransition; /***************************************************************************** * Optimizable Statements @@ -2105,6 +2120,8 @@ typedef struct CreateTrigStmt List *columns; /* column names, or NIL for all columns */ Node *whenClause; /* qual expression, or NULL if none */ bool isconstraint; /* This is a constraint trigger */ + /* explicitly named transition data */ + List *transitionRels; /* TriggerTransition nodes, or NIL if none */ /* The remaining fields are only used for constraint triggers */ bool deferrable; /* [NOT] DEFERRABLE */ bool initdeferred; /* INITIALLY {DEFERRED|IMMEDIATE} */ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 17ffef53a7..77d873beca 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -251,6 +251,7 @@ PG_KEYWORD("names", NAMES, UNRESERVED_KEYWORD) PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD) PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD) +PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD) PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD) PG_KEYWORD("no", NO, UNRESERVED_KEYWORD) PG_KEYWORD("none", NONE, COL_NAME_KEYWORD) @@ -268,6 +269,7 @@ PG_KEYWORD("of", OF, UNRESERVED_KEYWORD) PG_KEYWORD("off", OFF, UNRESERVED_KEYWORD) PG_KEYWORD("offset", OFFSET, RESERVED_KEYWORD) PG_KEYWORD("oids", OIDS, UNRESERVED_KEYWORD) +PG_KEYWORD("old", OLD, UNRESERVED_KEYWORD) PG_KEYWORD("on", ON, RESERVED_KEYWORD) PG_KEYWORD("only", ONLY, RESERVED_KEYWORD) PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD) @@ -313,6 +315,7 @@ PG_KEYWORD("recheck", RECHECK, UNRESERVED_KEYWORD) PG_KEYWORD("recursive", RECURSIVE, UNRESERVED_KEYWORD) PG_KEYWORD("ref", REF, UNRESERVED_KEYWORD) PG_KEYWORD("references", REFERENCES, RESERVED_KEYWORD) +PG_KEYWORD("referencing", REFERENCING, UNRESERVED_KEYWORD) PG_KEYWORD("refresh", REFRESH, UNRESERVED_KEYWORD) PG_KEYWORD("reindex", REINDEX, UNRESERVED_KEYWORD) PG_KEYWORD("relative", RELATIVE_P, UNRESERVED_KEYWORD) diff --git a/src/include/utils/reltrigger.h b/src/include/utils/reltrigger.h index e87f2283ec..756b417128 100644 --- a/src/include/utils/reltrigger.h +++ b/src/include/utils/reltrigger.h @@ -39,6 +39,8 @@ typedef struct Trigger int16 *tgattr; char **tgargs; char *tgqual; + char *tgoldtable; + char *tgnewtable; } Trigger; typedef struct TriggerDesc @@ -68,6 +70,11 @@ typedef struct TriggerDesc /* there are no row-level truncate triggers */ bool trig_truncate_before_statement; bool trig_truncate_after_statement; + /* Is there at least one trigger specifying each transition relation? */ + bool trig_insert_new_table; + bool trig_update_old_table; + bool trig_update_new_table; + bool trig_delete_old_table; } TriggerDesc; #endif /* RELTRIGGER_H */ From 927d7bb6b120a2ca09a164898f887eb850b7a329 Mon Sep 17 00:00:00 2001 From: Kevin Grittner Date: Fri, 4 Nov 2016 11:02:07 -0500 Subject: [PATCH 432/871] Improve tab completion for CREATE TRIGGER. This includes support for the new REFERENCING clause. --- src/bin/psql/tab-complete.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index dd8bbe1467..a43bbc519c 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2209,9 +2209,43 @@ psql_completion(const char *text, int start, int end) /* complete CREATE TRIGGER ... INSTEAD OF event ON with a list of views */ else if (TailMatches7("CREATE", "TRIGGER", MatchAny, "INSTEAD", "OF", MatchAny, "ON")) COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_views, NULL); + else if (HeadMatches2("CREATE", "TRIGGER") && TailMatches2("ON", MatchAny)) + COMPLETE_WITH_LIST7("NOT DEFERRABLE", "DEFERRABLE", "INITIALLY", + "REFERENCING", "FOR", "WHEN (", "EXECUTE PROCEDURE"); + else if (HeadMatches2("CREATE", "TRIGGER") && + (TailMatches1("DEFERRABLE") || TailMatches2("INITIALLY", "IMMEDIATE|DEFERRED"))) + COMPLETE_WITH_LIST4("REFERENCING", "FOR", "WHEN (", "EXECUTE PROCEDURE"); + else if (HeadMatches2("CREATE", "TRIGGER") && TailMatches1("REFERENCING")) + COMPLETE_WITH_LIST2("OLD TABLE", "NEW TABLE"); + else if (HeadMatches2("CREATE", "TRIGGER") && TailMatches2("OLD|NEW", "TABLE")) + COMPLETE_WITH_CONST("AS"); + else if (HeadMatches2("CREATE", "TRIGGER") && + (TailMatches5("REFERENCING", "OLD", "TABLE", "AS", MatchAny) || + TailMatches4("REFERENCING", "OLD", "TABLE", MatchAny))) + COMPLETE_WITH_LIST4("NEW TABLE", "FOR", "WHEN (", "EXECUTE PROCEDURE"); + else if (HeadMatches2("CREATE", "TRIGGER") && + (TailMatches5("REFERENCING", "NEW", "TABLE", "AS", MatchAny) || + TailMatches4("REFERENCING", "NEW", "TABLE", MatchAny))) + COMPLETE_WITH_LIST4("OLD TABLE", "FOR", "WHEN (", "EXECUTE PROCEDURE"); + else if (HeadMatches2("CREATE", "TRIGGER") && + (TailMatches9("REFERENCING", "OLD|NEW", "TABLE", "AS", MatchAny, "OLD|NEW", "TABLE", "AS", MatchAny) || + TailMatches8("REFERENCING", "OLD|NEW", "TABLE", MatchAny, "OLD|NEW", "TABLE", "AS", MatchAny) || + TailMatches8("REFERENCING", "OLD|NEW", "TABLE", "AS", MatchAny, "OLD|NEW", "TABLE", MatchAny) || + TailMatches7("REFERENCING", "OLD|NEW", "TABLE", MatchAny, "OLD|NEW", "TABLE", MatchAny))) + COMPLETE_WITH_LIST3("FOR", "WHEN (", "EXECUTE PROCEDURE"); + else if (HeadMatches2("CREATE", "TRIGGER") && TailMatches1("FOR")) + COMPLETE_WITH_LIST3("EACH", "ROW", "STATEMENT"); + else if (HeadMatches2("CREATE", "TRIGGER") && TailMatches2("FOR", "EACH")) + COMPLETE_WITH_LIST2("ROW", "STATEMENT"); + else if (HeadMatches2("CREATE", "TRIGGER") && + (TailMatches3("FOR", "EACH", "ROW|STATEMENT") || + TailMatches2("FOR", "ROW|STATEMENT"))) + COMPLETE_WITH_LIST2("WHEN (", "EXECUTE PROCEDURE"); /* complete CREATE TRIGGER ... EXECUTE with PROCEDURE */ else if (HeadMatches2("CREATE", "TRIGGER") && TailMatches1("EXECUTE")) COMPLETE_WITH_CONST("PROCEDURE"); + else if (HeadMatches2("CREATE", "TRIGGER") && TailMatches2("EXECUTE", "PROCEDURE")) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_functions, NULL); /* CREATE ROLE,USER,GROUP */ else if (Matches3("CREATE", "ROLE|GROUP|USER", MatchAny) && From 367b99bbb13a14bd96bba6f73c231b2056ba9592 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Nov 2016 12:11:47 -0400 Subject: [PATCH 433/871] Fix gin_leafpage_items(). On closer inspection, commit 84ad68d64 broke gin_leafpage_items(), because the aligned copy of the page got palloc'd in a short-lived context whereas it needs to be in the SRF's multi_call_memory_ctx. This was not exposed by the regression test, because the regression test doesn't actually exercise the function in a meaningful way. Fix the code bug, and extend the test in what I hope is a portable fashion. --- contrib/pageinspect/expected/gin.out | 11 +++++++++-- contrib/pageinspect/ginfuncs.c | 6 +++--- contrib/pageinspect/sql/gin.sql | 9 +++++++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/contrib/pageinspect/expected/gin.out b/contrib/pageinspect/expected/gin.out index 4f341e18d2..82f63b23b1 100644 --- a/contrib/pageinspect/expected/gin.out +++ b/contrib/pageinspect/expected/gin.out @@ -1,6 +1,6 @@ CREATE TABLE test1 (x int, y int[]); INSERT INTO test1 VALUES (1, ARRAY[11, 111]); -CREATE INDEX test1_y_idx ON test1 USING gin (y); +CREATE INDEX test1_y_idx ON test1 USING gin (y) WITH (fastupdate = off); \x SELECT * FROM gin_metapage_info(get_raw_page('test1_y_idx', 0)); -[ RECORD 1 ]----+----------- @@ -27,4 +27,11 @@ flags | {leaf} SELECT * FROM gin_leafpage_items(get_raw_page('test1_y_idx', 1)); ERROR: input page is not a compressed GIN data leaf page DETAIL: Flags 0002, expected 0083 -DROP TABLE test1; +INSERT INTO test1 SELECT x, ARRAY[1,10] FROM generate_series(2,10000) x; +SELECT COUNT(*) > 0 +FROM gin_leafpage_items(get_raw_page('test1_y_idx', + (pg_relation_size('test1_y_idx') / + current_setting('block_size')::bigint)::int - 1)); +-[ RECORD 1 ] +?column? | t + diff --git a/contrib/pageinspect/ginfuncs.c b/contrib/pageinspect/ginfuncs.c index 6e4103df6a..55285893f7 100644 --- a/contrib/pageinspect/ginfuncs.c +++ b/contrib/pageinspect/ginfuncs.c @@ -197,6 +197,9 @@ gin_leafpage_items(PG_FUNCTION_ARGS) Page page; GinPageOpaque opaq; + fctx = SRF_FIRSTCALL_INIT(); + mctx = MemoryContextSwitchTo(fctx->multi_call_memory_ctx); + page = get_page_from_raw(raw_page); if (PageGetSpecialSize(page) != MAXALIGN(sizeof(GinPageOpaqueData))) @@ -216,9 +219,6 @@ gin_leafpage_items(PG_FUNCTION_ARGS) opaq->flags, (GIN_DATA | GIN_LEAF | GIN_COMPRESSED)))); - fctx = SRF_FIRSTCALL_INIT(); - mctx = MemoryContextSwitchTo(fctx->multi_call_memory_ctx); - inter_call_data = palloc(sizeof(gin_leafpage_items_state)); /* Build a tuple descriptor for our result type */ diff --git a/contrib/pageinspect/sql/gin.sql b/contrib/pageinspect/sql/gin.sql index ba79ff2108..d516ed3cbd 100644 --- a/contrib/pageinspect/sql/gin.sql +++ b/contrib/pageinspect/sql/gin.sql @@ -1,6 +1,6 @@ CREATE TABLE test1 (x int, y int[]); INSERT INTO test1 VALUES (1, ARRAY[11, 111]); -CREATE INDEX test1_y_idx ON test1 USING gin (y); +CREATE INDEX test1_y_idx ON test1 USING gin (y) WITH (fastupdate = off); \x @@ -11,4 +11,9 @@ SELECT * FROM gin_page_opaque_info(get_raw_page('test1_y_idx', 1)); SELECT * FROM gin_leafpage_items(get_raw_page('test1_y_idx', 1)); -DROP TABLE test1; +INSERT INTO test1 SELECT x, ARRAY[1,10] FROM generate_series(2,10000) x; + +SELECT COUNT(*) > 0 +FROM gin_leafpage_items(get_raw_page('test1_y_idx', + (pg_relation_size('test1_y_idx') / + current_setting('block_size')::bigint)::int - 1)); From d5f6f13f8e7eb1c28395807922246294da4f57bb Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Nov 2016 13:26:49 -0400 Subject: [PATCH 434/871] Be more consistent about masking xl_info with ~XLR_INFO_MASK. Generally, WAL resource managers are only supposed to examine the top 4 bits of a WAL record's xl_info; the rest are reserved for the WAL mechanism itself. A few places were not consistent about doing this with respect to XLOG_CHECKPOINT and XLOG_SWITCH records. There's no bug currently, since no additional bits ever get set in these specific record types, but that might not be true forever. Let's follow the generic coding rule here too. Michael Paquier --- src/backend/access/transam/xlog.c | 13 ++++++++----- src/backend/access/transam/xlogreader.c | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 6b1f24ef1f..813f6467fd 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -903,8 +903,9 @@ XLogInsertRecord(XLogRecData *rdata, XLogRecPtr fpw_lsn) pg_crc32c rdata_crc; bool inserted; XLogRecord *rechdr = (XLogRecord *) rdata->data; + uint8 info = rechdr->xl_info & ~XLR_INFO_MASK; bool isLogSwitch = (rechdr->xl_rmid == RM_XLOG_ID && - rechdr->xl_info == XLOG_SWITCH); + info == XLOG_SWITCH); XLogRecPtr StartPos; XLogRecPtr EndPos; @@ -6170,7 +6171,7 @@ StartupXLOG(void) if (record != NULL) { memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - wasShutdown = (record->xl_info == XLOG_CHECKPOINT_SHUTDOWN); + wasShutdown = ((record->xl_info & ~XLR_INFO_MASK) == XLOG_CHECKPOINT_SHUTDOWN); ereport(DEBUG1, (errmsg("checkpoint record is at %X/%X", (uint32) (checkPointLoc >> 32), (uint32) checkPointLoc))); @@ -6328,7 +6329,7 @@ StartupXLOG(void) (errmsg("could not locate a valid checkpoint record"))); } memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - wasShutdown = (record->xl_info == XLOG_CHECKPOINT_SHUTDOWN); + wasShutdown = ((record->xl_info & ~XLR_INFO_MASK) == XLOG_CHECKPOINT_SHUTDOWN); } /* @@ -7785,6 +7786,7 @@ ReadCheckpointRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int whichChkpt, bool report) { XLogRecord *record; + uint8 info; if (!XRecOffIsValid(RecPtr)) { @@ -7810,6 +7812,7 @@ ReadCheckpointRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, } record = ReadRecord(xlogreader, RecPtr, LOG, true); + info = record->xl_info & ~XLR_INFO_MASK; if (record == NULL) { @@ -7852,8 +7855,8 @@ ReadCheckpointRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, } return NULL; } - if (record->xl_info != XLOG_CHECKPOINT_SHUTDOWN && - record->xl_info != XLOG_CHECKPOINT_ONLINE) + if (info != XLOG_CHECKPOINT_SHUTDOWN && + info != XLOG_CHECKPOINT_ONLINE) { switch (whichChkpt) { diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index f2da505892..56d4c66ebb 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -462,7 +462,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) /* * Special processing if it's an XLOG SWITCH record */ - if (record->xl_rmid == RM_XLOG_ID && record->xl_info == XLOG_SWITCH) + if (record->xl_rmid == RM_XLOG_ID && + (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH) { /* Pretend it extends to end of segment */ state->EndRecPtr += XLogSegSize - 1; From 20540710e83f2873707c284a0c0693f0b57156c4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Nov 2016 18:29:53 -0400 Subject: [PATCH 435/871] Delete contrib/xml2's legacy implementation of xml_is_well_formed(). This function is unreferenced in modern usage; it was superseded in 9.1 by a core function of the same name. It has been left in place in the C code only so that pre-9.1 SQL definitions of the contrib/xml2 functions would continue to work. Six years seems like enough time for people to have updated to the extension-style version of the xml2 module, so let's drop this. The key reason for not keeping it any longer is that we want to stick an explicit PGDLLEXPORT into PG_FUNCTION_INFO_V1(), and the similarity of name to the core function creates a conflict that compilers will complain about. Extracted from a larger patch for that purpose. I'm committing this change separately to give it more visibility in the commit logs. While at it, remove the documentation entry that claimed that xml_is_well_formed() is a function provided by contrib/xml2, and instead mention the even more ancient alias xml_valid(). Laurenz Albe, doc change by me Patch: --- contrib/xml2/xpath.c | 45 ------------------------------------------ doc/src/sgml/xml2.sgml | 10 +++++----- 2 files changed, 5 insertions(+), 50 deletions(-) diff --git a/contrib/xml2/xpath.c b/contrib/xml2/xpath.c index ac28996867..28445be369 100644 --- a/contrib/xml2/xpath.c +++ b/contrib/xml2/xpath.c @@ -81,51 +81,6 @@ pgxml_parser_init(PgXmlStrictness strictness) } -/* - * Returns true if document is well-formed - * - * Note: this has been superseded by a core function. We still have to - * have it in the contrib module so that existing SQL-level references - * to the function won't fail; but in normal usage with up-to-date SQL - * definitions for the contrib module, this won't be called. - */ - -PG_FUNCTION_INFO_V1(xml_is_well_formed); - -Datum -xml_is_well_formed(PG_FUNCTION_ARGS) -{ - text *t = PG_GETARG_TEXT_P(0); /* document buffer */ - bool result = false; - int32 docsize = VARSIZE(t) - VARHDRSZ; - xmlDocPtr doctree; - PgXmlErrorContext *xmlerrcxt; - - xmlerrcxt = pgxml_parser_init(PG_XML_STRICTNESS_LEGACY); - - PG_TRY(); - { - doctree = xmlParseMemory((char *) VARDATA(t), docsize); - - result = (doctree != NULL); - - if (doctree != NULL) - xmlFreeDoc(doctree); - } - PG_CATCH(); - { - pg_xml_done(xmlerrcxt, true); - - PG_RE_THROW(); - } - PG_END_TRY(); - - pg_xml_done(xmlerrcxt, false); - - PG_RETURN_BOOL(result); -} - - /* Encodes special characters (<, >, &, " and \r) as XML entities */ PG_FUNCTION_INFO_V1(xml_encode_special_chars); diff --git a/doc/src/sgml/xml2.sgml b/doc/src/sgml/xml2.sgml index a40172c36d..9bbc9e75d7 100644 --- a/doc/src/sgml/xml2.sgml +++ b/doc/src/sgml/xml2.sgml @@ -53,7 +53,7 @@ - xml_is_well_formed(document) + xml_valid(document) @@ -62,10 +62,10 @@ This parses the document text in its parameter and returns true if the - document is well-formed XML. (Note: before PostgreSQL 8.2, this - function was called xml_valid(). That is the wrong name - since validity and well-formedness have different meanings in XML. - The old name is still available, but is deprecated.) + document is well-formed XML. (Note: this is an alias for the standard + PostgreSQL function xml_is_well_formed(). The + name xml_valid() is technically incorrect since validity + and well-formedness have different meanings in XML.) From c8ead2a3974d3eada145a0e18940150039493cc9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Nov 2016 19:04:56 -0400 Subject: [PATCH 436/871] Provide DLLEXPORT markers for C functions via PG_FUNCTION_INFO_V1 macro. Second try at the change originally made in commit 8518583cd; this time with contrib updates so that manual extern declarations are also marked with PGDLLEXPORT. The release notes should point this out as a significant source-code change for extension authors, since they'll have to make similar additions to avoid trouble on Windows. Laurenz Albe, doc change by me Patch: --- contrib/hstore/hstore.h | 2 +- contrib/ltree/ltree.h | 40 ++++++++++++++++++++-------------------- doc/src/sgml/xfunc.sgml | 17 +++++++++++++++++ src/include/fmgr.h | 7 +++---- 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/contrib/hstore/hstore.h b/contrib/hstore/hstore.h index 6bab08b7de..6303fa4061 100644 --- a/contrib/hstore/hstore.h +++ b/contrib/hstore/hstore.h @@ -194,7 +194,7 @@ extern Pairs *hstoreArrayToPairs(ArrayType *a, int *npairs); #if HSTORE_POLLUTE_NAMESPACE #define HSTORE_POLLUTE(newname_,oldname_) \ PG_FUNCTION_INFO_V1(oldname_); \ - Datum newname_(PG_FUNCTION_ARGS); \ + extern PGDLLEXPORT Datum newname_(PG_FUNCTION_ARGS); \ Datum oldname_(PG_FUNCTION_ARGS) { return newname_(fcinfo); } \ extern int no_such_variable #else diff --git a/contrib/ltree/ltree.h b/contrib/ltree/ltree.h index c604357dbf..c7aa7f8818 100644 --- a/contrib/ltree/ltree.h +++ b/contrib/ltree/ltree.h @@ -130,30 +130,30 @@ typedef struct /* use in array iterator */ -Datum ltree_isparent(PG_FUNCTION_ARGS); -Datum ltree_risparent(PG_FUNCTION_ARGS); -Datum ltq_regex(PG_FUNCTION_ARGS); -Datum ltq_rregex(PG_FUNCTION_ARGS); -Datum lt_q_regex(PG_FUNCTION_ARGS); -Datum lt_q_rregex(PG_FUNCTION_ARGS); -Datum ltxtq_exec(PG_FUNCTION_ARGS); -Datum ltxtq_rexec(PG_FUNCTION_ARGS); -Datum _ltq_regex(PG_FUNCTION_ARGS); -Datum _ltq_rregex(PG_FUNCTION_ARGS); -Datum _lt_q_regex(PG_FUNCTION_ARGS); -Datum _lt_q_rregex(PG_FUNCTION_ARGS); -Datum _ltxtq_exec(PG_FUNCTION_ARGS); -Datum _ltxtq_rexec(PG_FUNCTION_ARGS); -Datum _ltree_isparent(PG_FUNCTION_ARGS); -Datum _ltree_risparent(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_isparent(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_risparent(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltq_regex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltq_rregex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum lt_q_regex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum lt_q_rregex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltxtq_exec(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltxtq_rexec(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltq_regex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltq_rregex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _lt_q_regex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _lt_q_rregex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltxtq_exec(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltxtq_rexec(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltree_isparent(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltree_risparent(PG_FUNCTION_ARGS); /* Concatenation functions */ -Datum ltree_addltree(PG_FUNCTION_ARGS); -Datum ltree_addtext(PG_FUNCTION_ARGS); -Datum ltree_textadd(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_addltree(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_addtext(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_textadd(PG_FUNCTION_ARGS); /* Util function */ -Datum ltree_in(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_in(PG_FUNCTION_ARGS); bool ltree_execute(ITEM *curitem, void *checkval, bool calcnot, bool (*chkcond) (void *checkval, ITEM *val)); diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index de6a466efc..6060e61857 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -2577,6 +2577,23 @@ concat_text(PG_FUNCTION_ARGS) error messages to this effect. + + + + To work correctly on Windows, C-language functions need + to be marked with PGDLLEXPORT, unless you use a build + process that marks all global functions that way. In simple cases + this detail will be handled transparently by + the PG_FUNCTION_INFO_V1 macro. However, if you write + explicit external declarations (perhaps in header files), be sure + to write them like this: + +extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); + + or you'll get compiler complaints when building on Windows. (On + other platforms, the PGDLLEXPORT macro does nothing.) + +
diff --git a/src/include/fmgr.h b/src/include/fmgr.h index 0878418516..3668ac3f6e 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -350,12 +350,11 @@ typedef const Pg_finfo_record *(*PGFInfoFunction) (void); * * On Windows, the function and info function must be exported. Our normal * build processes take care of that via .DEF files or --export-all-symbols. - * Module authors using a different build process might need to manually - * declare the function PGDLLEXPORT. We do that automatically here for the - * info function, since authors shouldn't need to be explicitly aware of it. + * Module authors using a different build process might do it differently, + * so we declare these functions PGDLLEXPORT for their convenience. */ #define PG_FUNCTION_INFO_V1(funcname) \ -extern Datum funcname(PG_FUNCTION_ARGS); \ +extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); \ extern PGDLLEXPORT const Pg_finfo_record * CppConcat(pg_finfo_,funcname)(void); \ const Pg_finfo_record * \ CppConcat(pg_finfo_,funcname) (void) \ From 6feb69f6cef8b1bd2829700e25e402f22e86f3bd Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 4 Nov 2016 12:00:00 -0400 Subject: [PATCH 437/871] doc: Port page header customizations to XSLT --- doc/src/sgml/stylesheet.xsl | 138 ++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/doc/src/sgml/stylesheet.xsl b/doc/src/sgml/stylesheet.xsl index 39c9df28ad..42e8cce368 100644 --- a/doc/src/sgml/stylesheet.xsl +++ b/doc/src/sgml/stylesheet.xsl @@ -246,4 +246,142 @@ set toc,title + + + + + + + + + + + + + + + + + + From d49cc588ca589cd378b5862fa5704eaade4a1380 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 4 Nov 2016 12:00:00 -0400 Subject: [PATCH 438/871] doc: Don't reformat .fo files before processing by fop This messes up the whitespace in the output PDF document in some places. --- doc/src/sgml/Makefile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/Makefile b/doc/src/sgml/Makefile index 5df2f04dd6..84c94e8ae0 100644 --- a/doc/src/sgml/Makefile +++ b/doc/src/sgml/Makefile @@ -270,20 +270,16 @@ htmlhelp: stylesheet-hh.xsl postgres.xml $(XMLLINT) --noout --valid postgres.xml $(XSLTPROC) $(XSLTPROCFLAGS) $^ -%-A4.fo.tmp: stylesheet-fo.xsl %.xml +%-A4.fo: stylesheet-fo.xsl %.xml $(XMLLINT) --noout --valid $*.xml $(XSLTPROC) $(XSLTPROCFLAGS) --stringparam paper.type A4 -o $@ $^ -%-US.fo.tmp: stylesheet-fo.xsl %.xml +%-US.fo: stylesheet-fo.xsl %.xml $(XMLLINT) --noout --valid $*.xml $(XSLTPROC) $(XSLTPROCFLAGS) --stringparam paper.type USletter -o $@ $^ FOP = fop -# reformat FO output so that locations of errors are easier to find -%.fo: %.fo.tmp - $(XMLLINT) --format --output $@ $^ - .SECONDARY: postgres-A4.fo postgres-US.fo %-fop.pdf: %.fo @@ -404,7 +400,7 @@ clean: # index rm -f HTML.index $(GENERATED_SGML) # XSLT - rm -f postgres.xml postgres.xmltmp htmlhelp.hhp toc.hhc index.hhk *.fo *.fo.tmp + rm -f postgres.xml postgres.xmltmp htmlhelp.hhp toc.hhc index.hhk *.fo # EPUB rm -f postgres.epub # Texinfo From 06f5fd2f4f3d0991af07456e7cf6632b4ad9d7e7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 5 Nov 2016 11:14:10 -0400 Subject: [PATCH 439/871] pgwin32_is_junction's argument should be "const char *" not "char *". We're passing const strings to it in places, and that's not an unreasonable thing to do. Per buildfarm (noted on frogmouth in particular). --- src/include/port.h | 2 +- src/port/dirmod.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/port.h b/src/include/port.h index b81fa4a89e..8a63958535 100644 --- a/src/include/port.h +++ b/src/include/port.h @@ -251,7 +251,7 @@ extern int pgunlink(const char *path); #if defined(WIN32) && !defined(__CYGWIN__) extern int pgsymlink(const char *oldpath, const char *newpath); extern int pgreadlink(const char *path, char *buf, size_t size); -extern bool pgwin32_is_junction(char *path); +extern bool pgwin32_is_junction(const char *path); #define symlink(oldpath, newpath) pgsymlink(oldpath, newpath) #define readlink(path, buf, size) pgreadlink(path, buf, size) diff --git a/src/port/dirmod.c b/src/port/dirmod.c index fe2b815ff6..aaf496df66 100644 --- a/src/port/dirmod.c +++ b/src/port/dirmod.c @@ -338,10 +338,10 @@ pgreadlink(const char *path, char *buf, size_t size) /* * Assumes the file exists, so will return false if it doesn't - * (since a nonexistant file is not a junction) + * (since a nonexistent file is not a junction) */ bool -pgwin32_is_junction(char *path) +pgwin32_is_junction(const char *path) { DWORD attr = GetFileAttributes(path); From 86d19d27ce6d588ebb0afa84f9121515fa11686d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 5 Nov 2016 11:51:46 -0400 Subject: [PATCH 440/871] Remove duplicate macro definition. Seems to be a copy-and-pasteo. Odd that we heard no reports of compiler warnings about it. Thomas Munro --- src/include/access/xact.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/include/access/xact.h b/src/include/access/xact.h index 503ae1b82d..a123d2a7c4 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -225,7 +225,6 @@ typedef struct xl_xact_twophase { TransactionId xid; } xl_xact_twophase; -#define MinSizeOfXactInvals offsetof(xl_xact_invals, msgs) typedef struct xl_xact_origin { From 34ca0905706422c191b3b0afef6e1c5f54399833 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 5 Nov 2016 13:48:11 -0400 Subject: [PATCH 441/871] Adjust cost_merge_append() to reflect use of binaryheap_replace_first(). Commit 7a2fe9bd0 improved merge append so that replacement of a tuple takes log(N) operations, not twice log(N). Since cost_merge_append knew about that explicitly, we should adjust it. This probably makes little difference in practice, but the obsolete comment is confusing. Ideally this would have been put in in 9.3 with the underlying behavior change; but I'm not going to back-patch it, since there's some small chance of changing a plan choice that somebody's optimized for. Thomas Munro Discussion: --- src/backend/optimizer/path/costsize.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 2a49639f12..e42895dc31 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -1577,8 +1577,7 @@ cost_sort(Path *path, PlannerInfo *root, * at any given instant holds the next tuple from each stream. If there * are N streams, we need about N*log2(N) tuple comparisons to construct * the heap at startup, and then for each output tuple, about log2(N) - * comparisons to delete the top heap entry and another log2(N) comparisons - * to insert its successor from the same stream. + * comparisons to replace the top entry. * * (The effective value of N will drop once some of the input streams are * exhausted, but it seems unlikely to be worth trying to account for that.) @@ -1619,7 +1618,7 @@ cost_merge_append(Path *path, PlannerInfo *root, startup_cost += comparison_cost * N * logN; /* Per-tuple heap maintenance cost */ - run_cost += tuples * comparison_cost * 2.0 * logN; + run_cost += tuples * comparison_cost * logN; /* * Also charge a small amount (arbitrarily set equal to operator cost) per From 1b00dd0ea0f392b08fa50f9fcaf60e8f20d26dfd Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 5 Nov 2016 17:32:29 -0400 Subject: [PATCH 442/871] Improve minor error-handling details in pltcl. Don't ask Tcl_GetIndexFromObj to store an error message in the interpreter in cases where the next argument isn't necessarily one of the options we're asking it to check for. At best that is a waste of time, and at worst it might cause an inappropriate error result to get left behind. Be sure to check for valid syntax (ie, no command arguments) in pltcl_SPI_lastoid. Extracted from a larger and otherwise-unrelated patch. Jim Nasby Patch: --- src/pl/tcl/pltcl.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index d236890490..9d72f47f59 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -2138,7 +2138,7 @@ pltcl_SPI_execute(ClientData cdata, Tcl_Interp *interp, i = 1; while (i < objc) { - if (Tcl_GetIndexFromObj(interp, objv[i], options, "option", + if (Tcl_GetIndexFromObj(NULL, objv[i], options, NULL, TCL_EXACT, &optIndex) != TCL_OK) break; @@ -2484,7 +2484,7 @@ pltcl_SPI_execute_plan(ClientData cdata, Tcl_Interp *interp, i = 1; while (i < objc) { - if (Tcl_GetIndexFromObj(interp, objv[i], options, "option", + if (Tcl_GetIndexFromObj(NULL, objv[i], options, NULL, TCL_EXACT, &optIndex) != TCL_OK) break; @@ -2667,6 +2667,15 @@ static int pltcl_SPI_lastoid(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) { + /* + * Check call syntax + */ + if (objc != 1) + { + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(SPI_lastoid)); return TCL_OK; } From 32416b0f9a8502e7dff8afbf78e494c091b045e8 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Nov 2016 10:45:58 -0500 Subject: [PATCH 443/871] More zic cleanup. The workaround the IANA guys chose to get rid of the clang warning we'd silenced in commit 23ed2ba81 turns out not to satisfy Coverity. Go back to the previous solution, ie, remove the useless comparison to SIZE_MAX. (In principle, there could be machines out there where it's not useless because ptrdiff_t is wider than size_t. But the whole thing is pretty academic anyway, as we could never approach this limit for any sane estimate of the amount of data that zic will ever be asked to work with.) Also, s/lineno/lineno_t/g, because if we accept their decision to start using "lineno" as a typedef, it is going to have very unpleasant consequences in our next pgindent run. Noted that while fooling with pltcl yesterday. --- src/timezone/zic.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/timezone/zic.c b/src/timezone/zic.c index 7da95aad53..2f623e03a5 100644 --- a/src/timezone/zic.c +++ b/src/timezone/zic.c @@ -48,13 +48,13 @@ static ptrdiff_t const PTRDIFF_MAX = MAXVAL(ptrdiff_t, TYPE_BIT(ptrdiff_t)); #endif /* The type and printf format for line numbers. */ -typedef int lineno; +typedef int lineno_t; #define PRIdLINENO "d" struct rule { const char *r_filename; - lineno r_linenum; + lineno_t r_linenum; const char *r_name; zic_t r_loyear; /* for example, 1986 */ @@ -91,7 +91,7 @@ struct rule struct zone { const char *z_filename; - lineno z_linenum; + lineno_t z_linenum; const char *z_name; zic_t z_gmtoff; @@ -169,7 +169,7 @@ static int leapcnt; static bool leapseen; static zic_t leapminyear; static zic_t leapmaxyear; -static lineno linenum; +static lineno_t linenum; static int max_abbrvar_len = PERCENT_Z_LEN_BOUND; static int max_format_len; static zic_t max_year; @@ -178,7 +178,7 @@ static bool noise; static bool print_abbrevs; static zic_t print_cutoff; static const char *rfilename; -static lineno rlinenum; +static lineno_t rlinenum; static const char *progname; static ptrdiff_t timecnt; static ptrdiff_t timecnt_alloc; @@ -276,7 +276,7 @@ static ptrdiff_t nzones_alloc; struct link { const char *l_filename; - lineno l_linenum; + lineno_t l_linenum; const char *l_from; const char *l_to; }; @@ -430,14 +430,13 @@ ecpyalloc(char const * str) } static void * -growalloc(void *ptr, size_t itemsize, ptrdiff_t nitems, ptrdiff_t * nitems_alloc) +growalloc(void *ptr, size_t itemsize, ptrdiff_t nitems, ptrdiff_t *nitems_alloc) { if (nitems < *nitems_alloc) return ptr; else { - ptrdiff_t nitems_max = PTRDIFF_MAX - WORK_AROUND_QTBUG_53071; - ptrdiff_t amax = nitems_max < SIZE_MAX ? nitems_max : SIZE_MAX; + ptrdiff_t amax = PTRDIFF_MAX - WORK_AROUND_QTBUG_53071; if ((amax - 1) / 3 * 2 < *nitems_alloc) memory_exhausted(_("integer overflow")); @@ -451,7 +450,7 @@ growalloc(void *ptr, size_t itemsize, ptrdiff_t nitems, ptrdiff_t * nitems_alloc */ static void -eats(char const * name, lineno num, char const * rname, lineno rnum) +eats(char const * name, lineno_t num, char const * rname, lineno_t rnum) { filename = name; linenum = num; @@ -460,7 +459,7 @@ eats(char const * name, lineno num, char const * rname, lineno rnum) } static void -eat(char const * name, lineno num) +eat(char const * name, lineno_t num) { eats(name, num, NULL, -1); } @@ -1157,7 +1156,7 @@ infile(const char *name) const struct lookup *lp; int nfields; bool wantcont; - lineno num; + lineno_t num; char buf[BUFSIZ]; if (strcmp(name, "-") == 0) From 5485c99e7f507b2849ac675e9c10f34551f645b6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Nov 2016 11:29:40 -0500 Subject: [PATCH 444/871] Fix silly nil-pointer-dereference bug introduced in commit d5f6f13f8. Don't fetch record->xl_info before we've verified that record isn't NULL. Per Coverity. Michael Paquier --- src/backend/access/transam/xlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 813f6467fd..6cec02797a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -7812,7 +7812,6 @@ ReadCheckpointRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, } record = ReadRecord(xlogreader, RecPtr, LOG, true); - info = record->xl_info & ~XLR_INFO_MASK; if (record == NULL) { @@ -7855,6 +7854,7 @@ ReadCheckpointRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, } return NULL; } + info = record->xl_info & ~XLR_INFO_MASK; if (info != XLOG_CHECKPOINT_SHUTDOWN && info != XLOG_CHECKPOINT_ONLINE) { From fc8b81a291bf7e1acfcbd40ed344f323f1e93a94 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Nov 2016 12:09:36 -0500 Subject: [PATCH 445/871] Need to do SPI_push/SPI_pop around expression evaluation in plpgsql. We must do this in case the expression evaluation results in calling another plpgsql function (or, really, anything using SPI). I missed the need for this when I converted exec_cast_value() from doing a simple InputFunctionCall() to doing ExecEvalExpr() in commit 1345cc67b. There is a SPI_push_conditional in InputFunctionCall(), so that there was no bug before that. Per bug #14414 from Marcos Castedo. Add a regression test based on his example, which was that a plpgsql function in a domain check constraint didn't work when assigning to a domain-type variable within plpgsql. Report: <20161106010947.1387.66380@wrigleys.postgresql.org> --- src/pl/plpgsql/src/pl_exec.c | 4 ++++ src/test/regress/expected/plpgsql.out | 19 +++++++++++++++++++ src/test/regress/sql/plpgsql.sql | 22 ++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index 470cf935df..042b31fd77 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -6300,6 +6300,8 @@ exec_cast_value(PLpgSQL_execstate *estate, ExprContext *econtext = estate->eval_econtext; MemoryContext oldcontext; + SPI_push(); + oldcontext = MemoryContextSwitchTo(get_eval_mcontext(estate)); econtext->caseValue_datum = value; @@ -6313,6 +6315,8 @@ exec_cast_value(PLpgSQL_execstate *estate, cast_entry->cast_in_use = false; MemoryContextSwitchTo(oldcontext); + + SPI_pop(); } } diff --git a/src/test/regress/expected/plpgsql.out b/src/test/regress/expected/plpgsql.out index a2c36e44ba..147fb9f2bb 100644 --- a/src/test/regress/expected/plpgsql.out +++ b/src/test/regress/expected/plpgsql.out @@ -5643,3 +5643,22 @@ end; $$; ERROR: unhandled assertion CONTEXT: PL/pgSQL function inline_code_block line 3 at ASSERT +-- Test use of plpgsql in a domain check constraint (cf. bug #14414) +create function plpgsql_domain_check(val int) returns boolean as $$ +begin return val > 0; end +$$ language plpgsql immutable; +create domain plpgsql_domain as integer check(plpgsql_domain_check(value)); +do $$ +declare v_test plpgsql_domain; +begin + v_test := 1; +end; +$$; +do $$ +declare v_test plpgsql_domain := 1; +begin + v_test := 0; -- fail +end; +$$; +ERROR: value for domain plpgsql_domain violates check constraint "plpgsql_domain_check" +CONTEXT: PL/pgSQL function inline_code_block line 4 at assignment diff --git a/src/test/regress/sql/plpgsql.sql b/src/test/regress/sql/plpgsql.sql index 776f2292ea..49223ff2b9 100644 --- a/src/test/regress/sql/plpgsql.sql +++ b/src/test/regress/sql/plpgsql.sql @@ -4428,3 +4428,25 @@ exception when others then null; -- do nothing end; $$; + +-- Test use of plpgsql in a domain check constraint (cf. bug #14414) + +create function plpgsql_domain_check(val int) returns boolean as $$ +begin return val > 0; end +$$ language plpgsql immutable; + +create domain plpgsql_domain as integer check(plpgsql_domain_check(value)); + +do $$ +declare v_test plpgsql_domain; +begin + v_test := 1; +end; +$$; + +do $$ +declare v_test plpgsql_domain := 1; +begin + v_test := 0; -- fail +end; +$$; From fd2664dcb71102a5d66d2453182c010fb219496c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Nov 2016 14:43:13 -0500 Subject: [PATCH 446/871] Rationalize and document pltcl's handling of magic ".tupno" array element. For a very long time, pltcl's spi_exec and spi_execp commands have had a behavior of storing the current row number as an element of output arrays, but this was never documented. Fix that. For an equally long time, pltcl_trigger_handler had a behavior of silently ignoring ".tupno" as an output column name, evidently so that the result of spi_exec could be used directly as a trigger result tuple. Not sure how useful that really is, but in any case it's bad that it would break attempts to use ".tupno" as an actual column name. We can fix it by not checking for ".tupno" until after we check for a column name match. This comports with the effective behavior of spi_exec[p] that ".tupno" is only magic when you don't have an actual column named that. In passing, wordsmith the description of returning modified tuples from a pltcl trigger. Noted while working on Jim Nasby's patch to support composite results from pltcl. The inability to return trigger tuples using ".tupno" as a column name is a bug, so back-patch to all supported branches. --- doc/src/sgml/pltcl.sgml | 54 ++++++++++++++++++++++++++--------------- src/pl/tcl/pltcl.c | 23 ++++++++++++------ 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/doc/src/sgml/pltcl.sgml b/doc/src/sgml/pltcl.sgml index 805cc89dc9..52fc44940c 100644 --- a/doc/src/sgml/pltcl.sgml +++ b/doc/src/sgml/pltcl.sgml @@ -296,20 +296,22 @@ $$ LANGUAGE pltcl; If the command is a SELECT statement, the values of the result columns are placed into Tcl variables named after the columns. If the -array option is given, the column values are - instead stored into the named associative array, with the - column names used as array indexes. + instead stored into elements of the named associative array, with the + column names used as array indexes. In addition, the current row + number within the result (counting from zero) is stored into the array + element named .tupno, unless that name is + in use as a column name in the result.
If the command is a SELECT statement and no loop-body script is given, then only the first row of results are stored into - Tcl variables; remaining rows, if any, are ignored. No storing occurs - if the - query returns no rows. (This case can be detected by checking the - result of spi_exec.) For example: + Tcl variables or array elements; remaining rows, if any, are ignored. + No storing occurs if the query returns no rows. (This case can be + detected by checking the result of spi_exec.) + For example: spi_exec "SELECT count(*) AS cnt FROM pg_proc" - will set the Tcl variable $cnt to the number of rows in the pg_proc system catalog. @@ -317,15 +319,15 @@ spi_exec "SELECT count(*) AS cnt FROM pg_proc" If the optional loop-body argument is given, it is a piece of Tcl script that is executed once for each row in the query result. (loop-body is ignored if the given - command is not a SELECT.) The values of the current row's columns - are stored into Tcl variables before each iteration. For example: - + command is not a SELECT.) + The values of the current row's columns + are stored into Tcl variables or array elements before each iteration. + For example: spi_exec -array C "SELECT * FROM pg_class" { elog DEBUG "have table $C(relname)" } - will print a log message for every row of pg_class. This feature works similarly to other Tcl looping constructs; in particular continue and break work in the @@ -667,21 +669,35 @@ SELECT 'doesn''t' AS ret The return value from a trigger procedure can be one of the strings - OK or SKIP, or a list as returned by the - array get Tcl command. If the return value is OK, - the operation (INSERT/UPDATE/DELETE) that fired the trigger will proceed + OK or SKIP, or a list of column name/value pairs. + If the return value is OK, + the operation (INSERT/UPDATE/DELETE) + that fired the trigger will proceed normally. SKIP tells the trigger manager to silently suppress the operation for this row. If a list is returned, it tells PL/Tcl to - return a modified row to the trigger manager. This is only meaningful + return a modified row to the trigger manager; the contents of the + modified row are specified by the column names and values in the list. + Any columns not mentioned in the list are set to null. + Returning a modified row is only meaningful for row-level BEFORE INSERT or UPDATE - triggers for which the modified row will be inserted instead of the one + triggers, for which the modified row will be inserted instead of the one given in $NEW; or for row-level INSTEAD OF INSERT or UPDATE triggers where the returned row - is used to support INSERT RETURNING and - UPDATE RETURNING commands. The return value is ignored for - other types of triggers. + is used as the source data for INSERT RETURNING or + UPDATE RETURNING clauses. + In row-level BEFORE DELETE or INSTEAD + OF DELETE triggers, returning a modified row has the same + effect as returning OK, that is the operation proceeds. + The trigger return value is ignored for all other types of triggers. + + + The result list can be made from an array representation of the + modified tuple with the array get Tcl command. + + + Here's a little example trigger procedure that forces an integer value in a table to keep track of the number of updates that are performed on the diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 9d72f47f59..44fcf68054 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -1118,21 +1118,23 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) Oid typioparam; FmgrInfo finfo; - /************************************************************ - * Ignore ".tupno" pseudo elements (see pltcl_set_tuple_values) - ************************************************************/ - if (strcmp(ret_name, ".tupno") == 0) - continue; - /************************************************************ * Get the attribute number + * + * We silently ignore ".tupno", if it's present but doesn't match + * any actual output column. This allows direct use of a row + * returned by pltcl_set_tuple_values(). ************************************************************/ attnum = SPI_fnumber(tupdesc, ret_name); if (attnum == SPI_ERROR_NOATTRIBUTE) + { + if (strcmp(ret_name, ".tupno") == 0) + continue; ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("unrecognized attribute \"%s\"", ret_name))); + } if (attnum <= 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -2703,8 +2705,7 @@ pltcl_set_tuple_values(Tcl_Interp *interp, const char *arrayname, const char *nullname = NULL; /************************************************************ - * Prepare pointers for Tcl_SetVar2() below and in array - * mode set the .tupno element + * Prepare pointers for Tcl_SetVar2() below ************************************************************/ if (arrayname == NULL) { @@ -2715,6 +2716,12 @@ pltcl_set_tuple_values(Tcl_Interp *interp, const char *arrayname, { arrptr = &arrayname; nameptr = &attname; + + /* + * When outputting to an array, fill the ".tupno" element with the + * current tuple number. This will be overridden below if ".tupno" is + * in use as an actual field name in the rowtype. + */ Tcl_SetVar2Ex(interp, arrayname, ".tupno", Tcl_NewWideIntObj(tupno), 0); } From 2178cbf40d3d75d87ab9b55579ac1cb0621baeff Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Nov 2016 16:09:57 -0500 Subject: [PATCH 447/871] Modernize result-tuple construction in pltcl_trigger_handler(). Use Tcl_ListObjGetElements instead of Tcl_SplitList. Aside from being possibly more efficient in its own right, this means we are no longer responsible for freeing a malloc'd result array, so we can get rid of a PG_TRY/PG_CATCH block. Use heap_form_tuple instead of SPI_modifytuple. We don't need the extra generality of the latter, since we're always replacing all columns. Nor do we need its memory-context-munging, since at this point we're already out of the SPI environment. Per comparison of this code to tuple-building code submitted by Jim Nasby. I've abandoned the thought of merging the two cases into a single routine, but we may as well make the older code simpler and faster where we can. --- src/pl/tcl/pltcl.c | 167 ++++++++++++++++++++------------------------- 1 file changed, 73 insertions(+), 94 deletions(-) diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 44fcf68054..97d1f7ef7d 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -870,12 +870,11 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) Tcl_Obj *tcl_newtup; int tcl_rc; int i; - int *modattrs; - Datum *modvalues; - char *modnulls; - int ret_numvals; const char *result; - const char **ret_values; + int result_Objc; + Tcl_Obj **result_Objv; + Datum *values; + bool *nulls; /* Connect to SPI manager */ if (SPI_connect() != SPI_OK_CONNECT) @@ -1065,13 +1064,16 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) throw_tcl_error(interp, prodesc->user_proname); /************************************************************ - * The return value from the procedure might be one of - * the magic strings OK or SKIP or a list from array get. - * We can check for OK or SKIP without worrying about encoding. + * Exit SPI environment. ************************************************************/ if (SPI_finish() != SPI_OK_FINISH) elog(ERROR, "SPI_finish() failed"); + /************************************************************ + * The return value from the procedure might be one of + * the magic strings OK or SKIP, or a list from array get. + * We can check for OK or SKIP without worrying about encoding. + ************************************************************/ result = Tcl_GetStringResult(interp); if (strcmp(result, "OK") == 0) @@ -1080,108 +1082,85 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) return (HeapTuple) NULL; /************************************************************ - * Convert the result value from the Tcl interpreter - * and setup structures for SPI_modifytuple(); + * Otherwise, the return value should be a column name/value list + * specifying the modified tuple to return. ************************************************************/ - if (Tcl_SplitList(interp, result, - &ret_numvals, &ret_values) != TCL_OK) + if (Tcl_ListObjGetElements(interp, Tcl_GetObjResult(interp), + &result_Objc, &result_Objv) != TCL_OK) ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("could not split return value from trigger: %s", utf_u2e(Tcl_GetStringResult(interp))))); - /* Use a TRY to ensure ret_values will get freed */ - PG_TRY(); - { - if (ret_numvals % 2 != 0) - ereport(ERROR, - (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), - errmsg("trigger's return list must have even number of elements"))); + if (result_Objc % 2 != 0) + ereport(ERROR, + (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), + errmsg("trigger's return list must have even number of elements"))); - modattrs = (int *) palloc(tupdesc->natts * sizeof(int)); - modvalues = (Datum *) palloc(tupdesc->natts * sizeof(Datum)); - for (i = 0; i < tupdesc->natts; i++) - { - modattrs[i] = i + 1; - modvalues[i] = (Datum) NULL; - } + values = (Datum *) palloc0(tupdesc->natts * sizeof(Datum)); + nulls = (bool *) palloc(tupdesc->natts * sizeof(bool)); + memset(nulls, true, tupdesc->natts * sizeof(bool)); - modnulls = palloc(tupdesc->natts); - memset(modnulls, 'n', tupdesc->natts); + for (i = 0; i < result_Objc; i += 2) + { + char *ret_name = utf_u2e(Tcl_GetString(result_Objv[i])); + char *ret_value = utf_u2e(Tcl_GetString(result_Objv[i + 1])); + int attnum; + Oid typinput; + Oid typioparam; + FmgrInfo finfo; - for (i = 0; i < ret_numvals; i += 2) + /************************************************************ + * Get the attribute number + * + * We silently ignore ".tupno", if it's present but doesn't match + * any actual output column. This allows direct use of a row + * returned by pltcl_set_tuple_values(). + ************************************************************/ + attnum = SPI_fnumber(tupdesc, ret_name); + if (attnum == SPI_ERROR_NOATTRIBUTE) { - char *ret_name = utf_u2e(ret_values[i]); - char *ret_value = utf_u2e(ret_values[i + 1]); - int attnum; - Oid typinput; - Oid typioparam; - FmgrInfo finfo; - - /************************************************************ - * Get the attribute number - * - * We silently ignore ".tupno", if it's present but doesn't match - * any actual output column. This allows direct use of a row - * returned by pltcl_set_tuple_values(). - ************************************************************/ - attnum = SPI_fnumber(tupdesc, ret_name); - if (attnum == SPI_ERROR_NOATTRIBUTE) - { - if (strcmp(ret_name, ".tupno") == 0) - continue; - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_COLUMN), - errmsg("unrecognized attribute \"%s\"", - ret_name))); - } - if (attnum <= 0) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot set system attribute \"%s\"", - ret_name))); - - /************************************************************ - * Ignore dropped columns - ************************************************************/ - if (tupdesc->attrs[attnum - 1]->attisdropped) + if (strcmp(ret_name, ".tupno") == 0) continue; - - /************************************************************ - * Lookup the attribute type in the syscache - * for the input function - ************************************************************/ - getTypeInputInfo(tupdesc->attrs[attnum - 1]->atttypid, - &typinput, &typioparam); - fmgr_info(typinput, &finfo); - - /************************************************************ - * Set the attribute to NOT NULL and convert the contents - ************************************************************/ - modvalues[attnum - 1] = InputFunctionCall(&finfo, - ret_value, - typioparam, - tupdesc->attrs[attnum - 1]->atttypmod); - modnulls[attnum - 1] = ' '; + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("unrecognized attribute \"%s\"", + ret_name))); } + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot set system attribute \"%s\"", + ret_name))); - rettup = SPI_modifytuple(trigdata->tg_relation, rettup, tupdesc->natts, - modattrs, modvalues, modnulls); + /************************************************************ + * Ignore dropped columns + ************************************************************/ + if (tupdesc->attrs[attnum - 1]->attisdropped) + continue; - pfree(modattrs); - pfree(modvalues); - pfree(modnulls); + /************************************************************ + * Lookup the attribute type's input function + ************************************************************/ + getTypeInputInfo(tupdesc->attrs[attnum - 1]->atttypid, + &typinput, &typioparam); + fmgr_info(typinput, &finfo); - if (rettup == NULL) - elog(ERROR, "SPI_modifytuple() failed - RC = %d", SPI_result); - } - PG_CATCH(); - { - ckfree((char *) ret_values); - PG_RE_THROW(); + /************************************************************ + * Set the attribute to NOT NULL and convert the contents + ************************************************************/ + values[attnum - 1] = InputFunctionCall(&finfo, + ret_value, + typioparam, + tupdesc->attrs[attnum - 1]->atttypmod); + nulls[attnum - 1] = false; } - PG_END_TRY(); - ckfree((char *) ret_values); + + /* Build the modified tuple to return */ + rettup = heap_form_tuple(tupdesc, values, nulls); + + pfree(values); + pfree(nulls); return rettup; } From 26abb50c490dee191df21282bc940b94118550aa Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Nov 2016 17:56:05 -0500 Subject: [PATCH 448/871] Support PL/Tcl functions that return composite types and/or sets. Jim Nasby, rather heavily editorialized by me Patch: --- doc/src/sgml/pltcl.sgml | 73 ++++- src/pl/tcl/expected/pltcl_queries.out | 61 ++++ src/pl/tcl/expected/pltcl_setup.out | 13 + src/pl/tcl/pltcl.c | 430 ++++++++++++++++++++++---- src/pl/tcl/sql/pltcl_queries.sql | 33 ++ src/pl/tcl/sql/pltcl_setup.sql | 16 + 6 files changed, 545 insertions(+), 81 deletions(-) diff --git a/doc/src/sgml/pltcl.sgml b/doc/src/sgml/pltcl.sgml index 52fc44940c..8afaf4ad36 100644 --- a/doc/src/sgml/pltcl.sgml +++ b/doc/src/sgml/pltcl.sgml @@ -94,11 +94,11 @@ $$ LANGUAGE pltcl; The body of the function is simply a piece of Tcl script. - When the function is called, the argument values are passed as - variables $1 ... $n to the - Tcl script. The result is returned - from the Tcl code in the usual way, with a return - statement. + When the function is called, the argument values are passed to the + Tcl script as variables named 1 + ... n. The result is + returned from the Tcl code in the usual way, with + a return statement. @@ -173,17 +173,57 @@ $$ LANGUAGE pltcl; - There is currently no support for returning a composite-type - result value, nor for returning sets. + PL/Tcl functions can return composite-type results, too. To do this, + the Tcl code must return a list of column name/value pairs matching + the expected result type. Any column names omitted from the list + are returned as nulls, and an error is raised if there are unexpected + column names. Here is an example: + + +CREATE FUNCTION square_cube(in int, out squared int, out cubed int) AS $$ + return [list squared [expr {$1 * $1}] cubed [expr {$1 * $1 * $1}]] +$$ LANGUAGE pltcl; + + + + The result list can be made from an array representation of the + desired tuple with the array get Tcl command. For example: + + +CREATE FUNCTION raise_pay(employee, delta int) RETURNS employee AS $$ + set 1(salary) [expr {$1(salary) + $2}] + return [array get 1] +$$ LANGUAGE pltcl; + + + + - PL/Tcl does not currently have full support for - domain types: it treats a domain the same as the underlying scalar - type. This means that constraints associated with the domain will - not be enforced. This is not an issue for function arguments, but - it is a hazard if you declare a PL/Tcl function - as returning a domain type. + PL/Tcl functions can return sets. To do this, the Tcl code should + call return_next once per row to be returned, + passing either the appropriate value when returning a scalar type, + or a list of column name/value pairs when returning a composite type. + Here is an example returning a scalar type: + + +CREATE FUNCTION sequence(int, int) RETURNS SETOF int AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next $i + } +$$ LANGUAGE pltcl; + + + and here is one returning a composite type: + + +CREATE FUNCTION table_of_squares(int, int) RETURNS TABLE (x int, x2 int) AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next [list x $i x2 [expr {$i * $i}]] + } +$$ LANGUAGE pltcl; + @@ -195,10 +235,9 @@ $$ LANGUAGE pltcl; The argument values supplied to a PL/Tcl function's code are simply the input arguments converted to text form (just as if they had been displayed by a SELECT statement). Conversely, the - return - command will accept any string that is acceptable input format for - the function's declared return type. So, within the PL/Tcl function, - all values are just text strings. + return and return_next commands will accept + any string that is acceptable input format for the function's declared + result type, or for the specified column of a composite result type. diff --git a/src/pl/tcl/expected/pltcl_queries.out b/src/pl/tcl/expected/pltcl_queries.out index 6cb1fdbb61..3a9fef3447 100644 --- a/src/pl/tcl/expected/pltcl_queries.out +++ b/src/pl/tcl/expected/pltcl_queries.out @@ -303,3 +303,64 @@ select tcl_lastoid('t2') > 0; t (1 row) +-- test some error cases +CREATE FUNCTION tcl_error(OUT a int, OUT b int) AS $$return {$$ LANGUAGE pltcl; +SELECT tcl_error(); +ERROR: missing close-brace +CREATE FUNCTION bad_record(OUT a text, OUT b text) AS $$return [list a]$$ LANGUAGE pltcl; +SELECT bad_record(); +ERROR: column name/value list must have even number of elements +CREATE FUNCTION bad_field(OUT a text, OUT b text) AS $$return [list a 1 b 2 cow 3]$$ LANGUAGE pltcl; +SELECT bad_field(); +ERROR: column name/value list contains nonexistent column name "cow" +-- test compound return +select * from tcl_test_cube_squared(5); + squared | cubed +---------+------- + 25 | 125 +(1 row) + +-- test SRF +select * from tcl_test_squared_rows(0,5); + x | y +---+---- + 0 | 0 + 1 | 1 + 2 | 4 + 3 | 9 + 4 | 16 +(5 rows) + +select * from tcl_test_sequence(0,5) as a; + a +--- + 0 + 1 + 2 + 3 + 4 +(5 rows) + +select 1, tcl_test_sequence(0,5); + ?column? | tcl_test_sequence +----------+------------------- + 1 | 0 + 1 | 1 + 1 | 2 + 1 | 3 + 1 | 4 +(5 rows) + +CREATE FUNCTION non_srf() RETURNS int AS $$return_next 1$$ LANGUAGE pltcl; +select non_srf(); +ERROR: return_next cannot be used in non-set-returning functions +CREATE FUNCTION bad_record_srf(OUT a text, OUT b text) RETURNS SETOF record AS $$ +return_next [list a] +$$ LANGUAGE pltcl; +SELECT bad_record_srf(); +ERROR: column name/value list must have even number of elements +CREATE FUNCTION bad_field_srf(OUT a text, OUT b text) RETURNS SETOF record AS $$ +return_next [list a 1 b 2 cow 3] +$$ LANGUAGE pltcl; +SELECT bad_field_srf(); +ERROR: column name/value list contains nonexistent column name "cow" diff --git a/src/pl/tcl/expected/pltcl_setup.out b/src/pl/tcl/expected/pltcl_setup.out index e65e9e3ff7..ed99d9b492 100644 --- a/src/pl/tcl/expected/pltcl_setup.out +++ b/src/pl/tcl/expected/pltcl_setup.out @@ -555,6 +555,19 @@ NOTICE: tclsnitch: ddl_command_start DROP TABLE NOTICE: tclsnitch: ddl_command_end DROP TABLE drop event trigger tcl_a_snitch; drop event trigger tcl_b_snitch; +CREATE FUNCTION tcl_test_cube_squared(in int, out squared int, out cubed int) AS $$ + return [list squared [expr {$1 * $1}] cubed [expr {$1 * $1 * $1}]] +$$ language pltcl; +CREATE FUNCTION tcl_test_squared_rows(int,int) RETURNS TABLE (x int, y int) AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next [list y [expr {$i * $i}] x $i] + } +$$ language pltcl; +CREATE FUNCTION tcl_test_sequence(int,int) RETURNS SETOF int AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next $i + } +$$ language pltcl; -- test use of errorCode in error handling create function tcl_error_handling_test() returns text as $$ global errorCode diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 97d1f7ef7d..3d529c2e7d 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -21,6 +21,7 @@ #include "commands/trigger.h" #include "executor/spi.h" #include "fmgr.h" +#include "funcapi.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -123,6 +124,9 @@ typedef struct pltcl_interp_desc * problem to manage its memory when we replace a proc definition. We do * not clean up pltcl_proc_descs when a pg_proc row is deleted, only when * it is updated, and the same policy applies to Tcl's copy as well.) + * + * Note that the data in this struct is shared across all active calls; + * nothing except the fn_refcount should be changed by a call instance. **********************************************************************/ typedef struct pltcl_proc_desc { @@ -137,6 +141,8 @@ typedef struct pltcl_proc_desc pltcl_interp_desc *interp_desc; /* interpreter to use */ FmgrInfo result_in_func; /* input function for fn's result type */ Oid result_typioparam; /* param to pass to same */ + bool fn_retisset; /* true if function returns a set */ + bool fn_retistuple; /* true if function returns composite */ int nargs; /* number of arguments */ /* these arrays have nargs entries: */ FmgrInfo *arg_out_func; /* output fns for arg types */ @@ -188,6 +194,32 @@ typedef struct pltcl_proc_ptr } pltcl_proc_ptr; +/********************************************************************** + * Per-call state + **********************************************************************/ +typedef struct pltcl_call_state +{ + /* Call info struct, or NULL in a trigger */ + FunctionCallInfo fcinfo; + + /* Function we're executing (NULL if not yet identified) */ + pltcl_proc_desc *prodesc; + + /* + * Information for SRFs and functions returning composite types. + * ret_tupdesc and attinmeta are set up if either fn_retistuple or + * fn_retisset, since even a scalar-returning SRF needs a tuplestore. + */ + TupleDesc ret_tupdesc; /* return rowtype, if retistuple or retisset */ + AttInMetadata *attinmeta; /* metadata for building tuples of that type */ + + ReturnSetInfo *rsi; /* passed-in ReturnSetInfo, if any */ + Tuplestorestate *tuple_store; /* SRFs accumulate result here */ + MemoryContext tuple_store_cxt; /* context and resowner for tuplestore */ + ResourceOwner tuple_store_owner; +} pltcl_call_state; + + /********************************************************************** * Global data **********************************************************************/ @@ -196,9 +228,8 @@ static Tcl_Interp *pltcl_hold_interp = NULL; static HTAB *pltcl_interp_htab = NULL; static HTAB *pltcl_proc_htab = NULL; -/* these are saved and restored by pltcl_handler */ -static FunctionCallInfo pltcl_current_fcinfo = NULL; -static pltcl_proc_desc *pltcl_current_prodesc = NULL; +/* this is saved and restored by pltcl_handler */ +static pltcl_call_state *pltcl_current_call_state = NULL; /********************************************************************** * Lookup table for SQLSTATE condition names @@ -225,10 +256,12 @@ static void pltcl_init_load_unknown(Tcl_Interp *interp); static Datum pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted); -static Datum pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted); - -static HeapTuple pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted); -static void pltcl_event_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted); +static Datum pltcl_func_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted); +static HeapTuple pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted); +static void pltcl_event_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted); static void throw_tcl_error(Tcl_Interp *interp, const char *proname); @@ -246,7 +279,8 @@ static int pltcl_argisnull(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); static int pltcl_returnnull(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); - +static int pltcl_returnnext(ClientData cdata, Tcl_Interp *interp, + int objc, Tcl_Obj *const objv[]); static int pltcl_SPI_execute(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); static int pltcl_process_SPI_result(Tcl_Interp *interp, @@ -265,6 +299,10 @@ static int pltcl_SPI_lastoid(ClientData cdata, Tcl_Interp *interp, static void pltcl_set_tuple_values(Tcl_Interp *interp, const char *arrayname, uint64 tupno, HeapTuple tuple, TupleDesc tupdesc); static Tcl_Obj *pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc); +static HeapTuple pltcl_build_tuple_result(Tcl_Interp *interp, + Tcl_Obj **kvObjv, int kvObjc, + pltcl_call_state *call_state); +static void pltcl_init_tuple_store(pltcl_call_state *call_state); /* @@ -432,7 +470,8 @@ pltcl_init_interp(pltcl_interp_desc *interp_desc, bool pltrusted) pltcl_argisnull, NULL, NULL); Tcl_CreateObjCommand(interp, "return_null", pltcl_returnnull, NULL, NULL); - + Tcl_CreateObjCommand(interp, "return_next", + pltcl_returnnext, NULL, NULL); Tcl_CreateObjCommand(interp, "spi_exec", pltcl_SPI_execute, NULL, NULL); Tcl_CreateObjCommand(interp, "spi_prepare", @@ -625,29 +664,33 @@ pltclu_call_handler(PG_FUNCTION_ARGS) } +/********************************************************************** + * pltcl_handler() - Handler for function and trigger calls, for + * both trusted and untrusted interpreters. + **********************************************************************/ static Datum pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted) { Datum retval; - FunctionCallInfo save_fcinfo; - pltcl_proc_desc *save_prodesc; - pltcl_proc_desc *this_prodesc; + pltcl_call_state current_call_state; + pltcl_call_state *save_call_state; /* - * Ensure that static pointers are saved/restored properly + * Initialize current_call_state to nulls/zeroes; in particular, set its + * prodesc pointer to null. Anything that sets it non-null should + * increase the prodesc's fn_refcount at the same time. We'll decrease + * the refcount, and then delete the prodesc if it's no longer referenced, + * on the way out of this function. This ensures that prodescs live as + * long as needed even if somebody replaces the originating pg_proc row + * while they're executing. */ - save_fcinfo = pltcl_current_fcinfo; - save_prodesc = pltcl_current_prodesc; + memset(¤t_call_state, 0, sizeof(current_call_state)); /* - * Reset pltcl_current_prodesc to null. Anything that sets it non-null - * should increase the prodesc's fn_refcount at the same time. We'll - * decrease the refcount, and then delete the prodesc if it's no longer - * referenced, on the way out of this function. This ensures that - * prodescs live as long as needed even if somebody replaces the - * originating pg_proc row while they're executing. + * Ensure that static pointer is saved/restored properly */ - pltcl_current_prodesc = NULL; + save_call_state = pltcl_current_call_state; + pltcl_current_call_state = ¤t_call_state; PG_TRY(); { @@ -657,47 +700,46 @@ pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted) */ if (CALLED_AS_TRIGGER(fcinfo)) { - pltcl_current_fcinfo = NULL; - retval = PointerGetDatum(pltcl_trigger_handler(fcinfo, pltrusted)); + /* invoke the trigger handler */ + retval = PointerGetDatum(pltcl_trigger_handler(fcinfo, + ¤t_call_state, + pltrusted)); } else if (CALLED_AS_EVENT_TRIGGER(fcinfo)) { - pltcl_current_fcinfo = NULL; - pltcl_event_trigger_handler(fcinfo, pltrusted); + /* invoke the event trigger handler */ + pltcl_event_trigger_handler(fcinfo, ¤t_call_state, pltrusted); retval = (Datum) 0; } else { - pltcl_current_fcinfo = fcinfo; - retval = pltcl_func_handler(fcinfo, pltrusted); + /* invoke the regular function handler */ + current_call_state.fcinfo = fcinfo; + retval = pltcl_func_handler(fcinfo, ¤t_call_state, pltrusted); } } PG_CATCH(); { - /* Restore globals, then clean up the prodesc refcount if any */ - this_prodesc = pltcl_current_prodesc; - pltcl_current_fcinfo = save_fcinfo; - pltcl_current_prodesc = save_prodesc; - if (this_prodesc != NULL) + /* Restore static pointer, then clean up the prodesc refcount if any */ + pltcl_current_call_state = save_call_state; + if (current_call_state.prodesc != NULL) { - Assert(this_prodesc->fn_refcount > 0); - if (--this_prodesc->fn_refcount == 0) - MemoryContextDelete(this_prodesc->fn_cxt); + Assert(current_call_state.prodesc->fn_refcount > 0); + if (--current_call_state.prodesc->fn_refcount == 0) + MemoryContextDelete(current_call_state.prodesc->fn_cxt); } PG_RE_THROW(); } PG_END_TRY(); - /* Restore globals, then clean up the prodesc refcount if any */ + /* Restore static pointer, then clean up the prodesc refcount if any */ /* (We're being paranoid in case an error is thrown in context deletion) */ - this_prodesc = pltcl_current_prodesc; - pltcl_current_fcinfo = save_fcinfo; - pltcl_current_prodesc = save_prodesc; - if (this_prodesc != NULL) + pltcl_current_call_state = save_call_state; + if (current_call_state.prodesc != NULL) { - Assert(this_prodesc->fn_refcount > 0); - if (--this_prodesc->fn_refcount == 0) - MemoryContextDelete(this_prodesc->fn_cxt); + Assert(current_call_state.prodesc->fn_refcount > 0); + if (--current_call_state.prodesc->fn_refcount == 0) + MemoryContextDelete(current_call_state.prodesc->fn_cxt); } return retval; @@ -708,7 +750,8 @@ pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted) * pltcl_func_handler() - Handler for regular function calls **********************************************************************/ static Datum -pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted) +pltcl_func_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted) { pltcl_proc_desc *prodesc; Tcl_Interp *volatile interp; @@ -725,11 +768,32 @@ pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted) prodesc = compile_pltcl_function(fcinfo->flinfo->fn_oid, InvalidOid, false, pltrusted); - pltcl_current_prodesc = prodesc; + call_state->prodesc = prodesc; prodesc->fn_refcount++; interp = prodesc->interp_desc->interp; + /* + * If we're a SRF, check caller can handle materialize mode, and save + * relevant info into call_state. We must ensure that the returned + * tuplestore is owned by the caller's context, even if we first create it + * inside a subtransaction. + */ + if (prodesc->fn_retisset) + { + ReturnSetInfo *rsi = (ReturnSetInfo *) fcinfo->resultinfo; + + if (!rsi || !IsA(rsi, ReturnSetInfo) || + (rsi->allowedModes & SFRM_Materialize) == 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + + call_state->rsi = rsi; + call_state->tuple_store_cxt = rsi->econtext->ecxt_per_query_memory; + call_state->tuple_store_owner = CurrentResourceOwner; + } + /************************************************************ * Create the tcl command to call the internal * proc in the Tcl interpreter @@ -838,11 +902,72 @@ pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted) if (SPI_finish() != SPI_OK_FINISH) elog(ERROR, "SPI_finish() failed"); - if (fcinfo->isnull) + if (prodesc->fn_retisset) + { + ReturnSetInfo *rsi = call_state->rsi; + + /* We already checked this is OK */ + rsi->returnMode = SFRM_Materialize; + + /* If we produced any tuples, send back the result */ + if (call_state->tuple_store) + { + rsi->setResult = call_state->tuple_store; + if (call_state->ret_tupdesc) + { + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(call_state->tuple_store_cxt); + rsi->setDesc = CreateTupleDescCopy(call_state->ret_tupdesc); + MemoryContextSwitchTo(oldcxt); + } + } + retval = (Datum) 0; + fcinfo->isnull = true; + } + else if (fcinfo->isnull) + { retval = InputFunctionCall(&prodesc->result_in_func, NULL, prodesc->result_typioparam, -1); + } + else if (prodesc->fn_retistuple) + { + TupleDesc td; + HeapTuple tup; + Tcl_Obj *resultObj; + Tcl_Obj **resultObjv; + int resultObjc; + + /* + * Set up data about result type. XXX it's tempting to consider + * caching this in the prodesc, in the common case where the rowtype + * is determined by the function not the calling query. But we'd have + * to be able to deal with ADD/DROP/ALTER COLUMN events when the + * result type is a named composite type, so it's not exactly trivial. + * Maybe worth improving someday. + */ + if (get_call_result_type(fcinfo, NULL, &td) != TYPEFUNC_COMPOSITE) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("function returning record called in context " + "that cannot accept type record"))); + + Assert(!call_state->ret_tupdesc); + Assert(!call_state->attinmeta); + call_state->ret_tupdesc = td; + call_state->attinmeta = TupleDescGetAttInMetadata(td); + + /* Convert function result to tuple */ + resultObj = Tcl_GetObjResult(interp); + if (Tcl_ListObjGetElements(interp, resultObj, &resultObjc, &resultObjv) == TCL_ERROR) + throw_tcl_error(interp, prodesc->user_proname); + + tup = pltcl_build_tuple_result(interp, resultObjv, resultObjc, + call_state); + retval = HeapTupleGetDatum(tup); + } else retval = InputFunctionCall(&prodesc->result_in_func, utf_u2e(Tcl_GetStringResult(interp)), @@ -857,7 +982,8 @@ pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted) * pltcl_trigger_handler() - Handler for trigger calls **********************************************************************/ static HeapTuple -pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) +pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted) { pltcl_proc_desc *prodesc; Tcl_Interp *volatile interp; @@ -886,7 +1012,7 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) false, /* not an event trigger */ pltrusted); - pltcl_current_prodesc = prodesc; + call_state->prodesc = prodesc; prodesc->fn_refcount++; interp = prodesc->interp_desc->interp; @@ -1169,7 +1295,8 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) * pltcl_event_trigger_handler() - Handler for event trigger calls **********************************************************************/ static void -pltcl_event_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) +pltcl_event_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted) { pltcl_proc_desc *prodesc; Tcl_Interp *volatile interp; @@ -1185,7 +1312,7 @@ pltcl_event_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) prodesc = compile_pltcl_function(fcinfo->flinfo->fn_oid, InvalidOid, true, pltrusted); - pltcl_current_prodesc = prodesc; + call_state->prodesc = prodesc; prodesc->fn_refcount++; interp = prodesc->interp_desc->interp; @@ -1389,10 +1516,11 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, procStruct->prorettype); typeStruct = (Form_pg_type) GETSTRUCT(typeTup); - /* Disallow pseudotype result, except VOID */ + /* Disallow pseudotype result, except VOID and RECORD */ if (typeStruct->typtype == TYPTYPE_PSEUDO) { - if (procStruct->prorettype == VOIDOID) + if (procStruct->prorettype == VOIDOID || + procStruct->prorettype == RECORDOID) /* okay */ ; else if (procStruct->prorettype == TRIGGEROID || procStruct->prorettype == EVTTRIGGEROID) @@ -1406,16 +1534,15 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, format_type_be(procStruct->prorettype)))); } - if (typeStruct->typtype == TYPTYPE_COMPOSITE) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("PL/Tcl functions cannot return composite types"))); - fmgr_info_cxt(typeStruct->typinput, &(prodesc->result_in_func), proc_cxt); prodesc->result_typioparam = getTypeIOParam(typeTup); + prodesc->fn_retisset = procStruct->proretset; + prodesc->fn_retistuple = (procStruct->prorettype == RECORDOID || + typeStruct->typtype == TYPTYPE_COMPOSITE); + ReleaseSysCache(typeTup); } @@ -1914,7 +2041,7 @@ pltcl_argisnull(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) { int argno; - FunctionCallInfo fcinfo = pltcl_current_fcinfo; + FunctionCallInfo fcinfo = pltcl_current_call_state->fcinfo; /************************************************************ * Check call syntax @@ -1967,7 +2094,7 @@ static int pltcl_returnnull(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) { - FunctionCallInfo fcinfo = pltcl_current_fcinfo; + FunctionCallInfo fcinfo = pltcl_current_call_state->fcinfo; /************************************************************ * Check call syntax @@ -1998,6 +2125,95 @@ pltcl_returnnull(ClientData cdata, Tcl_Interp *interp, } +/********************************************************************** + * pltcl_returnnext() - Add a row to the result tuplestore in a SRF. + **********************************************************************/ +static int +pltcl_returnnext(ClientData cdata, Tcl_Interp *interp, + int objc, Tcl_Obj *const objv[]) +{ + pltcl_call_state *call_state = pltcl_current_call_state; + FunctionCallInfo fcinfo = call_state->fcinfo; + pltcl_proc_desc *prodesc = call_state->prodesc; + int result = TCL_OK; + MemoryContext tmpcxt; + MemoryContext oldcxt; + + /* + * Check that we're called as a set-returning function + */ + if (fcinfo == NULL) + { + Tcl_SetObjResult(interp, + Tcl_NewStringObj("return_next cannot be used in triggers", -1)); + return TCL_ERROR; + } + + if (!prodesc->fn_retisset) + { + Tcl_SetObjResult(interp, + Tcl_NewStringObj("return_next cannot be used in non-set-returning functions", -1)); + return TCL_ERROR; + } + + /* + * Check call syntax + */ + if (objc != 2) + { + Tcl_WrongNumArgs(interp, 1, objv, "result"); + return TCL_ERROR; + } + + /* Set up tuple store if first output row */ + if (call_state->tuple_store == NULL) + pltcl_init_tuple_store(call_state); + + /* Make short-lived context to run input functions in */ + tmpcxt = AllocSetContextCreate(CurrentMemoryContext, + "pltcl_returnnext", + ALLOCSET_SMALL_SIZES); + oldcxt = MemoryContextSwitchTo(tmpcxt); + + if (prodesc->fn_retistuple) + { + Tcl_Obj **rowObjv; + int rowObjc; + + /* result should be a list, so break it down */ + if (Tcl_ListObjGetElements(interp, objv[1], &rowObjc, &rowObjv) == TCL_ERROR) + result = TCL_ERROR; + else + { + HeapTuple tuple; + + SPI_push(); + tuple = pltcl_build_tuple_result(interp, rowObjv, rowObjc, + call_state); + tuplestore_puttuple(call_state->tuple_store, tuple); + SPI_pop(); + } + } + else + { + Datum retval; + bool isNull = false; + + retval = InputFunctionCall(&prodesc->result_in_func, + utf_u2e((char *) Tcl_GetString(objv[1])), + prodesc->result_typioparam, + -1); + tuplestore_putvalues(call_state->tuple_store, call_state->ret_tupdesc, + &retval, &isNull); + } + + MemoryContextSwitchTo(oldcxt); + MemoryContextDelete(tmpcxt); + + return result; +} + + /*---------- * Support for running SPI operations inside subtransactions * @@ -2164,7 +2380,7 @@ pltcl_SPI_execute(ClientData cdata, Tcl_Interp *interp, { UTF_BEGIN; spi_rc = SPI_execute(UTF_U2E(Tcl_GetString(objv[query_idx])), - pltcl_current_prodesc->fn_readonly, count); + pltcl_current_call_state->prodesc->fn_readonly, count); UTF_END; my_rc = pltcl_process_SPI_result(interp, @@ -2414,7 +2630,7 @@ pltcl_SPI_prepare(ClientData cdata, Tcl_Interp *interp, * Insert a hashtable entry for the plan and return * the key to the caller ************************************************************/ - query_hash = &pltcl_current_prodesc->interp_desc->query_hash; + query_hash = &pltcl_current_call_state->prodesc->interp_desc->query_hash; hashent = Tcl_CreateHashEntry(query_hash, qdesc->qname, &hashnew); Tcl_SetHashValue(hashent, (ClientData) qdesc); @@ -2503,7 +2719,7 @@ pltcl_SPI_execute_plan(ClientData cdata, Tcl_Interp *interp, return TCL_ERROR; } - query_hash = &pltcl_current_prodesc->interp_desc->query_hash; + query_hash = &pltcl_current_call_state->prodesc->interp_desc->query_hash; hashent = Tcl_FindHashEntry(query_hash, Tcl_GetString(objv[i])); if (hashent == NULL) @@ -2618,7 +2834,8 @@ pltcl_SPI_execute_plan(ClientData cdata, Tcl_Interp *interp, * Execute the plan ************************************************************/ spi_rc = SPI_execute_plan(qdesc->plan, argvalues, nulls, - pltcl_current_prodesc->fn_readonly, count); + pltcl_current_call_state->prodesc->fn_readonly, + count); my_rc = pltcl_process_SPI_result(interp, arrayname, @@ -2808,3 +3025,88 @@ pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc) return retobj; } + +/********************************************************************** + * pltcl_build_tuple_result() - Build a tuple of function's result rowtype + * from a Tcl list of column names and values + * + * Note: this function leaks memory. Even if we made it clean up its own + * mess, there's no way to prevent the datatype input functions it calls + * from leaking. Run it in a short-lived context, unless we're about to + * exit the procedure anyway. + * + * Also, caller is responsible for doing SPI_push/SPI_pop if calling from + * inside SPI environment. + **********************************************************************/ +static HeapTuple +pltcl_build_tuple_result(Tcl_Interp *interp, Tcl_Obj **kvObjv, int kvObjc, + pltcl_call_state *call_state) +{ + char **values; + int i; + + if (kvObjc % 2 != 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("column name/value list must have even number of elements"))); + + values = (char **) palloc0(call_state->ret_tupdesc->natts * sizeof(char *)); + + for (i = 0; i < kvObjc; i += 2) + { + char *fieldName = utf_e2u(Tcl_GetString(kvObjv[i])); + int attn = SPI_fnumber(call_state->ret_tupdesc, fieldName); + + if (attn <= 0 || call_state->ret_tupdesc->attrs[attn - 1]->attisdropped) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column name/value list contains nonexistent column name \"%s\"", + fieldName))); + + values[attn - 1] = utf_e2u(Tcl_GetString(kvObjv[i + 1])); + } + + return BuildTupleFromCStrings(call_state->attinmeta, values); +} + +/********************************************************************** + * pltcl_init_tuple_store() - Initialize the result tuplestore for a SRF + **********************************************************************/ +static void +pltcl_init_tuple_store(pltcl_call_state *call_state) +{ + ReturnSetInfo *rsi = call_state->rsi; + MemoryContext oldcxt; + ResourceOwner oldowner; + + /* Should be in a SRF */ + Assert(rsi); + /* Should be first time through */ + Assert(!call_state->tuple_store); + Assert(!call_state->attinmeta); + + /* We expect caller to provide an appropriate result tupdesc */ + Assert(rsi->expectedDesc); + call_state->ret_tupdesc = rsi->expectedDesc; + + /* + * Switch to the right memory context and resource owner for storing the + * tuplestore. If we're within a subtransaction opened for an exception + * block, for example, we must still create the tuplestore in the resource + * owner that was active when this function was entered, and not in the + * subtransaction's resource owner. + */ + oldcxt = MemoryContextSwitchTo(call_state->tuple_store_cxt); + oldowner = CurrentResourceOwner; + CurrentResourceOwner = call_state->tuple_store_owner; + + call_state->tuple_store = + tuplestore_begin_heap(rsi->allowedModes & SFRM_Materialize_Random, + false, work_mem); + + /* Build attinmeta in this context, too */ + call_state->attinmeta = TupleDescGetAttInMetadata(call_state->ret_tupdesc); + + CurrentResourceOwner = oldowner; + MemoryContextSwitchTo(oldcxt); +} diff --git a/src/pl/tcl/sql/pltcl_queries.sql b/src/pl/tcl/sql/pltcl_queries.sql index a0a9619a9b..0ebfe65340 100644 --- a/src/pl/tcl/sql/pltcl_queries.sql +++ b/src/pl/tcl/sql/pltcl_queries.sql @@ -97,3 +97,36 @@ create temp table t1 (f1 int); select tcl_lastoid('t1'); create temp table t2 (f1 int) with oids; select tcl_lastoid('t2') > 0; + +-- test some error cases +CREATE FUNCTION tcl_error(OUT a int, OUT b int) AS $$return {$$ LANGUAGE pltcl; +SELECT tcl_error(); + +CREATE FUNCTION bad_record(OUT a text, OUT b text) AS $$return [list a]$$ LANGUAGE pltcl; +SELECT bad_record(); + +CREATE FUNCTION bad_field(OUT a text, OUT b text) AS $$return [list a 1 b 2 cow 3]$$ LANGUAGE pltcl; +SELECT bad_field(); + +-- test compound return +select * from tcl_test_cube_squared(5); + +-- test SRF +select * from tcl_test_squared_rows(0,5); + +select * from tcl_test_sequence(0,5) as a; + +select 1, tcl_test_sequence(0,5); + +CREATE FUNCTION non_srf() RETURNS int AS $$return_next 1$$ LANGUAGE pltcl; +select non_srf(); + +CREATE FUNCTION bad_record_srf(OUT a text, OUT b text) RETURNS SETOF record AS $$ +return_next [list a] +$$ LANGUAGE pltcl; +SELECT bad_record_srf(); + +CREATE FUNCTION bad_field_srf(OUT a text, OUT b text) RETURNS SETOF record AS $$ +return_next [list a 1 b 2 cow 3] +$$ LANGUAGE pltcl; +SELECT bad_field_srf(); diff --git a/src/pl/tcl/sql/pltcl_setup.sql b/src/pl/tcl/sql/pltcl_setup.sql index 8df65a5816..58f38d53aa 100644 --- a/src/pl/tcl/sql/pltcl_setup.sql +++ b/src/pl/tcl/sql/pltcl_setup.sql @@ -596,6 +596,22 @@ drop table foo; drop event trigger tcl_a_snitch; drop event trigger tcl_b_snitch; +CREATE FUNCTION tcl_test_cube_squared(in int, out squared int, out cubed int) AS $$ + return [list squared [expr {$1 * $1}] cubed [expr {$1 * $1 * $1}]] +$$ language pltcl; + +CREATE FUNCTION tcl_test_squared_rows(int,int) RETURNS TABLE (x int, y int) AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next [list y [expr {$i * $i}] x $i] + } +$$ language pltcl; + +CREATE FUNCTION tcl_test_sequence(int,int) RETURNS SETOF int AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next $i + } +$$ language pltcl; + -- test use of errorCode in error handling create function tcl_error_handling_test() returns text as $$ From 7f1bcfb93df90adcf3c7230234fc123096e2a639 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Nov 2016 19:22:12 -0500 Subject: [PATCH 449/871] Sync pltcl_build_tuple_result's error handling with pltcl_trigger_handler. Meant to do this in 26abb50c4, but forgot. --- src/pl/tcl/pltcl.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 3d529c2e7d..3e52113ee2 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -3057,11 +3057,29 @@ pltcl_build_tuple_result(Tcl_Interp *interp, Tcl_Obj **kvObjv, int kvObjc, char *fieldName = utf_e2u(Tcl_GetString(kvObjv[i])); int attn = SPI_fnumber(call_state->ret_tupdesc, fieldName); - if (attn <= 0 || call_state->ret_tupdesc->attrs[attn - 1]->attisdropped) + /* + * As in pltcl_trigger_handler, silently ignore ".tupno" if it's in + * the list but doesn't match any column name. + */ + if (attn == SPI_ERROR_NOATTRIBUTE) + { + if (strcmp(fieldName, ".tupno") == 0) + continue; ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column name/value list contains nonexistent column name \"%s\"", fieldName))); + } + + if (attn <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot set system attribute \"%s\"", + fieldName))); + + /* Ignore dropped attributes */ + if (call_state->ret_tupdesc->attrs[attn - 1]->attisdropped) + continue; values[attn - 1] = utf_e2u(Tcl_GetString(kvObjv[i + 1])); } From a5954de1051a779f1460426ab88d142f8790f18d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 27 Oct 2016 12:00:00 -0400 Subject: [PATCH 450/871] Save redundant code for pseudotype I/O functions Use a macro to generate the in and out functions for pseudotypes that reject all input and output, saving many lines of redundant code. Parameterize the error messages to reduce translatable strings. --- src/backend/utils/adt/pseudotypes.c | 346 ++++------------------------ 1 file changed, 45 insertions(+), 301 deletions(-) diff --git a/src/backend/utils/adt/pseudotypes.c b/src/backend/utils/adt/pseudotypes.c index dd447cf4e8..4e8acb418c 100644 --- a/src/backend/utils/adt/pseudotypes.c +++ b/src/backend/utils/adt/pseudotypes.c @@ -83,34 +83,6 @@ cstring_send(PG_FUNCTION_ARGS) PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } - -/* - * any_in - input routine for pseudo-type ANY. - */ -Datum -any_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type any"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * any_out - output routine for pseudo-type ANY. - */ -Datum -any_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type any"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - - /* * anyarray_in - input routine for pseudo-type ANYARRAY. */ @@ -119,7 +91,7 @@ anyarray_in(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type anyarray"))); + errmsg("cannot accept a value of type %s", "anyarray"))); PG_RETURN_VOID(); /* keep compiler quiet */ } @@ -147,7 +119,7 @@ anyarray_recv(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type anyarray"))); + errmsg("cannot accept a value of type %s", "anyarray"))); PG_RETURN_VOID(); /* keep compiler quiet */ } @@ -172,7 +144,7 @@ anyenum_in(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type anyenum"))); + errmsg("cannot accept a value of type %s", "anyenum"))); PG_RETURN_VOID(); /* keep compiler quiet */ } @@ -196,7 +168,7 @@ anyrange_in(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type anyrange"))); + errmsg("cannot accept a value of type %s", "anyrange"))); PG_RETURN_VOID(); /* keep compiler quiet */ } @@ -264,275 +236,6 @@ void_send(PG_FUNCTION_ARGS) PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } - -/* - * trigger_in - input routine for pseudo-type TRIGGER. - */ -Datum -trigger_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type trigger"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * trigger_out - output routine for pseudo-type TRIGGER. - */ -Datum -trigger_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type trigger"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - - -/* - * event_trigger_in - input routine for pseudo-type event_trigger. - */ -Datum -event_trigger_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type event_trigger"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * event_trigger_out - output routine for pseudo-type event_trigger. - */ -Datum -event_trigger_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type event_trigger"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - - -/* - * language_handler_in - input routine for pseudo-type LANGUAGE_HANDLER. - */ -Datum -language_handler_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type language_handler"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * language_handler_out - output routine for pseudo-type LANGUAGE_HANDLER. - */ -Datum -language_handler_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type language_handler"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - - -/* - * fdw_handler_in - input routine for pseudo-type FDW_HANDLER. - */ -Datum -fdw_handler_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type fdw_handler"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * fdw_handler_out - output routine for pseudo-type FDW_HANDLER. - */ -Datum -fdw_handler_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type fdw_handler"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - - -/* - * index_am_handler_in - input routine for pseudo-type INDEX_AM_HANDLER. - */ -Datum -index_am_handler_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type index_am_handler"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * index_am_handler_out - output routine for pseudo-type INDEX_AM_HANDLER. - */ -Datum -index_am_handler_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type index_am_handler"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - - -/* - * tsm_handler_in - input routine for pseudo-type TSM_HANDLER. - */ -Datum -tsm_handler_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type tsm_handler"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * tsm_handler_out - output routine for pseudo-type TSM_HANDLER. - */ -Datum -tsm_handler_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type tsm_handler"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - - -/* - * internal_in - input routine for pseudo-type INTERNAL. - */ -Datum -internal_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type internal"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * internal_out - output routine for pseudo-type INTERNAL. - */ -Datum -internal_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type internal"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - - -/* - * opaque_in - input routine for pseudo-type OPAQUE. - */ -Datum -opaque_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type opaque"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * opaque_out - output routine for pseudo-type OPAQUE. - */ -Datum -opaque_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type opaque"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - - -/* - * anyelement_in - input routine for pseudo-type ANYELEMENT. - */ -Datum -anyelement_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type anyelement"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * anyelement_out - output routine for pseudo-type ANYELEMENT. - */ -Datum -anyelement_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type anyelement"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * anynonarray_in - input routine for pseudo-type ANYNONARRAY. - */ -Datum -anynonarray_in(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot accept a value of type anynonarray"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - -/* - * anynonarray_out - output routine for pseudo-type ANYNONARRAY. - */ -Datum -anynonarray_out(PG_FUNCTION_ARGS) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot display a value of type anynonarray"))); - - PG_RETURN_VOID(); /* keep compiler quiet */ -} - /* * shell_in - input routine for "shell" types (those not yet filled in). */ @@ -674,3 +377,44 @@ pg_ddl_command_send(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + + +/* + * Generate input and output functions for a pseudotype that will reject all + * input and output attempts. + */ +#define PSEUDOTYPE_DUMMY_IO_FUNCS(typname) \ +\ +Datum \ +typname##_in(PG_FUNCTION_ARGS) \ +{ \ + ereport(ERROR, \ + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ + errmsg("cannot accept a value of type %s", #typname))); \ +\ + PG_RETURN_VOID(); /* keep compiler quiet */ \ +} \ +\ +Datum \ +typname##_out(PG_FUNCTION_ARGS) \ +{ \ + ereport(ERROR, \ + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ + errmsg("cannot display a value of type %s", #typname))); \ +\ + PG_RETURN_VOID(); /* keep compiler quiet */ \ +} \ +\ +extern int no_such_variable + +PSEUDOTYPE_DUMMY_IO_FUNCS(any); +PSEUDOTYPE_DUMMY_IO_FUNCS(trigger); +PSEUDOTYPE_DUMMY_IO_FUNCS(event_trigger); +PSEUDOTYPE_DUMMY_IO_FUNCS(language_handler); +PSEUDOTYPE_DUMMY_IO_FUNCS(fdw_handler); +PSEUDOTYPE_DUMMY_IO_FUNCS(index_am_handler); +PSEUDOTYPE_DUMMY_IO_FUNCS(tsm_handler); +PSEUDOTYPE_DUMMY_IO_FUNCS(internal); +PSEUDOTYPE_DUMMY_IO_FUNCS(opaque); +PSEUDOTYPE_DUMMY_IO_FUNCS(anyelement); +PSEUDOTYPE_DUMMY_IO_FUNCS(anynonarray); From 48dbcbf22c69bd994ea5e3bf7be9a23b0606d80d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 7 Nov 2016 12:00:00 -0500 Subject: [PATCH 451/871] pg_rewing pg_upgrade: Fix translation markers In pg_log_v(), we need to translate the fmt before processing, not the formatted message afterwards. --- src/bin/pg_rewind/logging.c | 10 +++++----- src/bin/pg_upgrade/util.c | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/bin/pg_rewind/logging.c b/src/bin/pg_rewind/logging.c index a232abb39f..0ac1b7add4 100644 --- a/src/bin/pg_rewind/logging.c +++ b/src/bin/pg_rewind/logging.c @@ -34,26 +34,26 @@ pg_log_v(eLogType type, const char *fmt, va_list ap) { char message[QUERY_ALLOC]; - vsnprintf(message, sizeof(message), fmt, ap); + vsnprintf(message, sizeof(message), _(fmt), ap); switch (type) { case PG_DEBUG: if (debug) - printf("%s", _(message)); + printf("%s", message); break; case PG_PROGRESS: if (showprogress) - printf("%s", _(message)); + printf("%s", message); break; case PG_WARNING: - printf("%s", _(message)); + printf("%s", message); break; case PG_FATAL: - printf("\n%s", _(message)); + printf("\n%s", message); printf("%s", _("Failure, exiting\n")); exit(1); break; diff --git a/src/bin/pg_upgrade/util.c b/src/bin/pg_upgrade/util.c index aadc1cdd9d..52e891219b 100644 --- a/src/bin/pg_upgrade/util.c +++ b/src/bin/pg_upgrade/util.c @@ -89,7 +89,7 @@ pg_log_v(eLogType type, const char *fmt, va_list ap) { char message[QUERY_ALLOC]; - vsnprintf(message, sizeof(message), fmt, ap); + vsnprintf(message, sizeof(message), _(fmt), ap); /* PG_VERBOSE and PG_STATUS are only output in verbose mode */ /* fopen() on log_opts.internal might have failed, so check it */ @@ -108,7 +108,7 @@ pg_log_v(eLogType type, const char *fmt, va_list ap) { case PG_VERBOSE: if (log_opts.verbose) - printf("%s", _(message)); + printf("%s", message); break; case PG_STATUS: @@ -123,16 +123,16 @@ pg_log_v(eLogType type, const char *fmt, va_list ap) strlen(message) <= MESSAGE_WIDTH - 2 ? message : message + strlen(message) - MESSAGE_WIDTH + 3 + 2); else - printf(" %s\n", _(message)); + printf(" %s\n", message); break; case PG_REPORT: case PG_WARNING: - printf("%s", _(message)); + printf("%s", message); break; case PG_FATAL: - printf("\n%s", _(message)); + printf("\n%s", message); printf("Failure, exiting\n"); exit(1); break; From 77517ba59f8d3a9d282c5e826bf19cbe195cd784 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 14 Oct 2016 12:00:00 -0400 Subject: [PATCH 452/871] pg_upgrade: Add NLS Reviewed-by: Michael Paquier --- src/bin/pg_upgrade/function.c | 2 +- src/bin/pg_upgrade/info.c | 8 +-- src/bin/pg_upgrade/nls.mk | 12 ++++ src/bin/pg_upgrade/option.c | 101 +++++++++++++++---------------- src/bin/pg_upgrade/pg_upgrade.c | 1 + src/bin/pg_upgrade/relfilenode.c | 6 +- src/bin/pg_upgrade/server.c | 4 +- src/bin/pg_upgrade/util.c | 4 +- 8 files changed, 75 insertions(+), 63 deletions(-) create mode 100644 src/bin/pg_upgrade/nls.mk diff --git a/src/bin/pg_upgrade/function.c b/src/bin/pg_upgrade/function.c index 30093407da..5b60f9f459 100644 --- a/src/bin/pg_upgrade/function.c +++ b/src/bin/pg_upgrade/function.c @@ -252,7 +252,7 @@ check_loadable_libraries(void) if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) pg_fatal("could not open file \"%s\": %s\n", output_path, strerror(errno)); - fprintf(script, "could not load library \"%s\":\n%s\n", + fprintf(script, _("could not load library \"%s\":\n%s\n"), lib, PQerrorMessage(conn)); } diff --git a/src/bin/pg_upgrade/info.c b/src/bin/pg_upgrade/info.c index 1200c7fca2..8af9eacd28 100644 --- a/src/bin/pg_upgrade/info.c +++ b/src/bin/pg_upgrade/info.c @@ -238,7 +238,7 @@ report_unmatched_relation(const RelInfo *rel, const DbInfo *db, bool is_new_db) { snprintf(reldesc + strlen(reldesc), sizeof(reldesc) - strlen(reldesc), - " which is an index on \"%s.%s\"", + _(" which is an index on \"%s.%s\""), hrel->nspname, hrel->relname); /* Shift attention to index's table for toast check */ rel = hrel; @@ -248,7 +248,7 @@ report_unmatched_relation(const RelInfo *rel, const DbInfo *db, bool is_new_db) if (i >= db->rel_arr.nrels) snprintf(reldesc + strlen(reldesc), sizeof(reldesc) - strlen(reldesc), - " which is an index on OID %u", rel->indtable); + _(" which is an index on OID %u"), rel->indtable); } if (rel->toastheap) { @@ -260,7 +260,7 @@ report_unmatched_relation(const RelInfo *rel, const DbInfo *db, bool is_new_db) { snprintf(reldesc + strlen(reldesc), sizeof(reldesc) - strlen(reldesc), - " which is the TOAST table for \"%s.%s\"", + _(" which is the TOAST table for \"%s.%s\""), brel->nspname, brel->relname); break; } @@ -268,7 +268,7 @@ report_unmatched_relation(const RelInfo *rel, const DbInfo *db, bool is_new_db) if (i >= db->rel_arr.nrels) snprintf(reldesc + strlen(reldesc), sizeof(reldesc) - strlen(reldesc), - " which is the TOAST table for OID %u", rel->toastheap); + _(" which is the TOAST table for OID %u"), rel->toastheap); } if (is_new_db) diff --git a/src/bin/pg_upgrade/nls.mk b/src/bin/pg_upgrade/nls.mk new file mode 100644 index 0000000000..a0c846d9ea --- /dev/null +++ b/src/bin/pg_upgrade/nls.mk @@ -0,0 +1,12 @@ +# src/bin/pg_upgrade/nls.mk +CATALOG_NAME = pg_upgrade +AVAIL_LANGUAGES = +GETTEXT_FILES = check.c controldata.c dump.c exec.c file.c function.c \ + info.c option.c parallel.c pg_upgrade.c relfilenode.c \ + server.c tablespace.c util.c version.c +GETTEXT_TRIGGERS = pg_fatal pg_log:2 prep_status report_status:2 +GETTEXT_FLAGS = \ + pg_fatal:1:c-format \ + pg_log:2:c-format \ + prep_status:1:c-format \ + report_status:2:c-format diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c index 2e9a40c2b6..12a49ff990 100644 --- a/src/bin/pg_upgrade/option.c +++ b/src/bin/pg_upgrade/option.c @@ -240,13 +240,13 @@ parseCommandLine(int argc, char *argv[]) /* Get values from env if not already set */ check_required_directory(&old_cluster.bindir, NULL, "PGBINOLD", "-b", - "old cluster binaries reside"); + _("old cluster binaries reside")); check_required_directory(&new_cluster.bindir, NULL, "PGBINNEW", "-B", - "new cluster binaries reside"); + _("new cluster binaries reside")); check_required_directory(&old_cluster.pgdata, &old_cluster.pgconfig, - "PGDATAOLD", "-d", "old cluster data resides"); + "PGDATAOLD", "-d", _("old cluster data resides")); check_required_directory(&new_cluster.pgdata, &new_cluster.pgconfig, - "PGDATANEW", "-D", "new cluster data resides"); + "PGDATANEW", "-D", _("new cluster data resides")); #ifdef WIN32 @@ -275,56 +275,53 @@ parseCommandLine(int argc, char *argv[]) static void usage(void) { - printf(_("pg_upgrade upgrades a PostgreSQL cluster to a different major version.\n\ -\nUsage:\n\ - pg_upgrade [OPTION]...\n\ -\n\ -Options:\n\ - -b, --old-bindir=BINDIR old cluster executable directory\n\ - -B, --new-bindir=BINDIR new cluster executable directory\n\ - -c, --check check clusters only, don't change any data\n\ - -d, --old-datadir=DATADIR old cluster data directory\n\ - -D, --new-datadir=DATADIR new cluster data directory\n\ - -j, --jobs number of simultaneous processes or threads to use\n\ - -k, --link link instead of copying files to new cluster\n\ - -o, --old-options=OPTIONS old cluster options to pass to the server\n\ - -O, --new-options=OPTIONS new cluster options to pass to the server\n\ - -p, --old-port=PORT old cluster port number (default %d)\n\ - -P, --new-port=PORT new cluster port number (default %d)\n\ - -r, --retain retain SQL and log files after success\n\ - -U, --username=NAME cluster superuser (default \"%s\")\n\ - -v, --verbose enable verbose internal logging\n\ - -V, --version display version information, then exit\n\ - -?, --help show this help, then exit\n\ -\n\ -Before running pg_upgrade you must:\n\ - create a new database cluster (using the new version of initdb)\n\ - shutdown the postmaster servicing the old cluster\n\ - shutdown the postmaster servicing the new cluster\n\ -\n\ -When you run pg_upgrade, you must provide the following information:\n\ - the data directory for the old cluster (-d DATADIR)\n\ - the data directory for the new cluster (-D DATADIR)\n\ - the \"bin\" directory for the old version (-b BINDIR)\n\ - the \"bin\" directory for the new version (-B BINDIR)\n\ -\n\ -For example:\n\ - pg_upgrade -d oldCluster/data -D newCluster/data -b oldCluster/bin -B newCluster/bin\n\ -or\n"), old_cluster.port, new_cluster.port, os_info.user); + printf(_("pg_upgrade upgrades a PostgreSQL cluster to a different major version.\n\n")); + printf(_("Usage:\n")); + printf(_(" pg_upgrade [OPTION]...\n\n")); + printf(_("Options:\n")); + printf(_(" -b, --old-bindir=BINDIR old cluster executable directory\n")); + printf(_(" -B, --new-bindir=BINDIR new cluster executable directory\n")); + printf(_(" -c, --check check clusters only, don't change any data\n")); + printf(_(" -d, --old-datadir=DATADIR old cluster data directory\n")); + printf(_(" -D, --new-datadir=DATADIR new cluster data directory\n")); + printf(_(" -j, --jobs number of simultaneous processes or threads to use\n")); + printf(_(" -k, --link link instead of copying files to new cluster\n")); + printf(_(" -o, --old-options=OPTIONS old cluster options to pass to the server\n")); + printf(_(" -O, --new-options=OPTIONS new cluster options to pass to the server\n")); + printf(_(" -p, --old-port=PORT old cluster port number (default %d)\n"), old_cluster.port); + printf(_(" -P, --new-port=PORT new cluster port number (default %d)\n"), new_cluster.port); + printf(_(" -r, --retain retain SQL and log files after success\n")); + printf(_(" -U, --username=NAME cluster superuser (default \"%s\")\n"), os_info.user); + printf(_(" -v, --verbose enable verbose internal logging\n")); + printf(_(" -V, --version display version information, then exit\n")); + printf(_(" -?, --help show this help, then exit\n")); + printf(_("\n" + "Before running pg_upgrade you must:\n" + " create a new database cluster (using the new version of initdb)\n" + " shutdown the postmaster servicing the old cluster\n" + " shutdown the postmaster servicing the new cluster\n")); + printf(_("\n" + "When you run pg_upgrade, you must provide the following information:\n" + " the data directory for the old cluster (-d DATADIR)\n" + " the data directory for the new cluster (-D DATADIR)\n" + " the \"bin\" directory for the old version (-b BINDIR)\n" + " the \"bin\" directory for the new version (-B BINDIR)\n")); + printf(_("\n" + "For example:\n" + " pg_upgrade -d oldCluster/data -D newCluster/data -b oldCluster/bin -B newCluster/bin\n" + "or\n")); #ifndef WIN32 - printf(_("\ - $ export PGDATAOLD=oldCluster/data\n\ - $ export PGDATANEW=newCluster/data\n\ - $ export PGBINOLD=oldCluster/bin\n\ - $ export PGBINNEW=newCluster/bin\n\ - $ pg_upgrade\n")); + printf(_(" $ export PGDATAOLD=oldCluster/data\n" + " $ export PGDATANEW=newCluster/data\n" + " $ export PGBINOLD=oldCluster/bin\n" + " $ export PGBINNEW=newCluster/bin\n" + " $ pg_upgrade\n")); #else - printf(_("\ - C:\\> set PGDATAOLD=oldCluster/data\n\ - C:\\> set PGDATANEW=newCluster/data\n\ - C:\\> set PGBINOLD=oldCluster/bin\n\ - C:\\> set PGBINNEW=newCluster/bin\n\ - C:\\> pg_upgrade\n")); + printf(_(" C:\\> set PGDATAOLD=oldCluster/data\n" + " C:\\> set PGDATANEW=newCluster/data\n" + " C:\\> set PGBINOLD=oldCluster/bin\n" + " C:\\> set PGBINNEW=newCluster/bin\n" + " C:\\> pg_upgrade\n")); #endif printf(_("\nReport bugs to .\n")); } diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 90c07205bf..0207d852cf 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -75,6 +75,7 @@ main(int argc, char **argv) char *deletion_script_file_name = NULL; bool live_check = false; + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_upgrade")); parseCommandLine(argc, argv); get_restricted_token(os_info.progname); diff --git a/src/bin/pg_upgrade/relfilenode.c b/src/bin/pg_upgrade/relfilenode.c index c8c2a28f4e..79e41d1dec 100644 --- a/src/bin/pg_upgrade/relfilenode.c +++ b/src/bin/pg_upgrade/relfilenode.c @@ -30,8 +30,10 @@ void transfer_all_new_tablespaces(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr, char *old_pgdata, char *new_pgdata) { - pg_log(PG_REPORT, "%s user relation files\n", - user_opts.transfer_mode == TRANSFER_MODE_LINK ? "Linking" : "Copying"); + if (user_opts.transfer_mode == TRANSFER_MODE_LINK) + pg_log(PG_REPORT, "Linking user relation files\n"); + else + pg_log(PG_REPORT, "Copying user relation files\n"); /* * Transferring files by tablespace is tricky because a single database diff --git a/src/bin/pg_upgrade/server.c b/src/bin/pg_upgrade/server.c index 12432bb1d0..4892934c2e 100644 --- a/src/bin/pg_upgrade/server.c +++ b/src/bin/pg_upgrade/server.c @@ -36,7 +36,7 @@ connectToServer(ClusterInfo *cluster, const char *db_name) if (conn) PQfinish(conn); - printf("Failure, exiting\n"); + printf(_("Failure, exiting\n")); exit(1); } @@ -136,7 +136,7 @@ executeQueryOrDie(PGconn *conn, const char *fmt,...) PQerrorMessage(conn)); PQclear(result); PQfinish(conn); - printf("Failure, exiting\n"); + printf(_("Failure, exiting\n")); exit(1); } else diff --git a/src/bin/pg_upgrade/util.c b/src/bin/pg_upgrade/util.c index 52e891219b..4f27c72ba1 100644 --- a/src/bin/pg_upgrade/util.c +++ b/src/bin/pg_upgrade/util.c @@ -133,7 +133,7 @@ pg_log_v(eLogType type, const char *fmt, va_list ap) case PG_FATAL: printf("\n%s", message); - printf("Failure, exiting\n"); + printf(_("Failure, exiting\n")); exit(1); break; @@ -163,7 +163,7 @@ pg_fatal(const char *fmt,...) va_start(args, fmt); pg_log_v(PG_FATAL, fmt, args); va_end(args); - printf("Failure, exiting\n"); + printf(_("Failure, exiting\n")); exit(1); } From 33cb96ba1a84c612491fb5794674a649d1a6a4d6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 7 Nov 2016 10:19:22 -0500 Subject: [PATCH 453/871] Revert "Provide DLLEXPORT markers for C functions via PG_FUNCTION_INFO_V1 macro." This reverts commit c8ead2a3974d3eada145a0e18940150039493cc9. Seems there is no way to do this that doesn't cause MSVC to give warnings, so let's just go back to the way we've been doing it. Discussion: <11843.1478358206@sss.pgh.pa.us> --- contrib/hstore/hstore.h | 2 +- contrib/ltree/ltree.h | 40 ++++++++++++++++++++-------------------- doc/src/sgml/xfunc.sgml | 17 ----------------- src/include/fmgr.h | 7 ++++--- 4 files changed, 25 insertions(+), 41 deletions(-) diff --git a/contrib/hstore/hstore.h b/contrib/hstore/hstore.h index 6303fa4061..6bab08b7de 100644 --- a/contrib/hstore/hstore.h +++ b/contrib/hstore/hstore.h @@ -194,7 +194,7 @@ extern Pairs *hstoreArrayToPairs(ArrayType *a, int *npairs); #if HSTORE_POLLUTE_NAMESPACE #define HSTORE_POLLUTE(newname_,oldname_) \ PG_FUNCTION_INFO_V1(oldname_); \ - extern PGDLLEXPORT Datum newname_(PG_FUNCTION_ARGS); \ + Datum newname_(PG_FUNCTION_ARGS); \ Datum oldname_(PG_FUNCTION_ARGS) { return newname_(fcinfo); } \ extern int no_such_variable #else diff --git a/contrib/ltree/ltree.h b/contrib/ltree/ltree.h index c7aa7f8818..c604357dbf 100644 --- a/contrib/ltree/ltree.h +++ b/contrib/ltree/ltree.h @@ -130,30 +130,30 @@ typedef struct /* use in array iterator */ -extern PGDLLEXPORT Datum ltree_isparent(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltree_risparent(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltq_regex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltq_rregex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum lt_q_regex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum lt_q_rregex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltxtq_exec(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltxtq_rexec(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltq_regex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltq_rregex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _lt_q_regex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _lt_q_rregex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltxtq_exec(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltxtq_rexec(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltree_isparent(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltree_risparent(PG_FUNCTION_ARGS); +Datum ltree_isparent(PG_FUNCTION_ARGS); +Datum ltree_risparent(PG_FUNCTION_ARGS); +Datum ltq_regex(PG_FUNCTION_ARGS); +Datum ltq_rregex(PG_FUNCTION_ARGS); +Datum lt_q_regex(PG_FUNCTION_ARGS); +Datum lt_q_rregex(PG_FUNCTION_ARGS); +Datum ltxtq_exec(PG_FUNCTION_ARGS); +Datum ltxtq_rexec(PG_FUNCTION_ARGS); +Datum _ltq_regex(PG_FUNCTION_ARGS); +Datum _ltq_rregex(PG_FUNCTION_ARGS); +Datum _lt_q_regex(PG_FUNCTION_ARGS); +Datum _lt_q_rregex(PG_FUNCTION_ARGS); +Datum _ltxtq_exec(PG_FUNCTION_ARGS); +Datum _ltxtq_rexec(PG_FUNCTION_ARGS); +Datum _ltree_isparent(PG_FUNCTION_ARGS); +Datum _ltree_risparent(PG_FUNCTION_ARGS); /* Concatenation functions */ -extern PGDLLEXPORT Datum ltree_addltree(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltree_addtext(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltree_textadd(PG_FUNCTION_ARGS); +Datum ltree_addltree(PG_FUNCTION_ARGS); +Datum ltree_addtext(PG_FUNCTION_ARGS); +Datum ltree_textadd(PG_FUNCTION_ARGS); /* Util function */ -extern PGDLLEXPORT Datum ltree_in(PG_FUNCTION_ARGS); +Datum ltree_in(PG_FUNCTION_ARGS); bool ltree_execute(ITEM *curitem, void *checkval, bool calcnot, bool (*chkcond) (void *checkval, ITEM *val)); diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 6060e61857..de6a466efc 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -2577,23 +2577,6 @@ concat_text(PG_FUNCTION_ARGS) error messages to this effect.
- - - - To work correctly on Windows, C-language functions need - to be marked with PGDLLEXPORT, unless you use a build - process that marks all global functions that way. In simple cases - this detail will be handled transparently by - the PG_FUNCTION_INFO_V1 macro. However, if you write - explicit external declarations (perhaps in header files), be sure - to write them like this: - -extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); - - or you'll get compiler complaints when building on Windows. (On - other platforms, the PGDLLEXPORT macro does nothing.) - - diff --git a/src/include/fmgr.h b/src/include/fmgr.h index 3668ac3f6e..0878418516 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -350,11 +350,12 @@ typedef const Pg_finfo_record *(*PGFInfoFunction) (void); * * On Windows, the function and info function must be exported. Our normal * build processes take care of that via .DEF files or --export-all-symbols. - * Module authors using a different build process might do it differently, - * so we declare these functions PGDLLEXPORT for their convenience. + * Module authors using a different build process might need to manually + * declare the function PGDLLEXPORT. We do that automatically here for the + * info function, since authors shouldn't need to be explicitly aware of it. */ #define PG_FUNCTION_INFO_V1(funcname) \ -extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); \ +extern Datum funcname(PG_FUNCTION_ARGS); \ extern PGDLLEXPORT const Pg_finfo_record * CppConcat(pg_finfo_,funcname)(void); \ const Pg_finfo_record * \ CppConcat(pg_finfo_,funcname) (void) \ From c59f94e81e56fe24428952f116f3c9555f42cc4f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 7 Nov 2016 10:27:52 -0500 Subject: [PATCH 454/871] Revert "Delete contrib/xml2's legacy implementation of xml_is_well_formed()." This partly reverts commit 20540710e83f2873707c284a0c0693f0b57156c4. Since we've given up on adding PGDLLEXPORT markers to PG_FUNCTION_INFO_V1, there's no need to remove the legacy compatibility function. I kept the documentation changes, though, as they seem appropriate anyway. --- contrib/xml2/xpath.c | 45 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/contrib/xml2/xpath.c b/contrib/xml2/xpath.c index 28445be369..ac28996867 100644 --- a/contrib/xml2/xpath.c +++ b/contrib/xml2/xpath.c @@ -81,6 +81,51 @@ pgxml_parser_init(PgXmlStrictness strictness) } +/* + * Returns true if document is well-formed + * + * Note: this has been superseded by a core function. We still have to + * have it in the contrib module so that existing SQL-level references + * to the function won't fail; but in normal usage with up-to-date SQL + * definitions for the contrib module, this won't be called. + */ + +PG_FUNCTION_INFO_V1(xml_is_well_formed); + +Datum +xml_is_well_formed(PG_FUNCTION_ARGS) +{ + text *t = PG_GETARG_TEXT_P(0); /* document buffer */ + bool result = false; + int32 docsize = VARSIZE(t) - VARHDRSZ; + xmlDocPtr doctree; + PgXmlErrorContext *xmlerrcxt; + + xmlerrcxt = pgxml_parser_init(PG_XML_STRICTNESS_LEGACY); + + PG_TRY(); + { + doctree = xmlParseMemory((char *) VARDATA(t), docsize); + + result = (doctree != NULL); + + if (doctree != NULL) + xmlFreeDoc(doctree); + } + PG_CATCH(); + { + pg_xml_done(xmlerrcxt, true); + + PG_RE_THROW(); + } + PG_END_TRY(); + + pg_xml_done(xmlerrcxt, false); + + PG_RETURN_BOOL(result); +} + + /* Encodes special characters (<, >, &, " and \r) as XML entities */ PG_FUNCTION_INFO_V1(xml_encode_special_chars); From e3e66d8a9813d22c2aa027d8f373a96d4d4c1b15 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 7 Nov 2016 12:08:18 -0500 Subject: [PATCH 455/871] Band-aid fix for incorrect use of view options as StdRdOptions. We really ought to make StdRdOptions and the other decoded forms of reloptions self-identifying, but for the moment, assume that only plain relations could possibly be user_catalog_tables. Fixes problem with bogus "ON CONFLICT is not supported on table ... used as a catalog table" error when target is a view with cascade option. Discussion: <26681.1477940227@sss.pgh.pa.us> --- src/include/utils/rel.h | 3 ++- src/test/regress/expected/insert_conflict.out | 24 +++++++++++++++++++ src/test/regress/sql/insert_conflict.sql | 18 ++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index ed14442cfe..c867ebb233 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -270,7 +270,8 @@ typedef struct StdRdOptions * from the pov of logical decoding. Note multiple eval of argument! */ #define RelationIsUsedAsCatalogTable(relation) \ - ((relation)->rd_options ? \ + ((relation)->rd_rel->relkind == RELKIND_RELATION && \ + (relation)->rd_options ? \ ((StdRdOptions *) (relation)->rd_options)->user_catalog_table : false) /* diff --git a/src/test/regress/expected/insert_conflict.out b/src/test/regress/expected/insert_conflict.out index 8d8d69b1ad..63859c53ac 100644 --- a/src/test/regress/expected/insert_conflict.out +++ b/src/test/regress/expected/insert_conflict.out @@ -471,6 +471,30 @@ on conflict (b) where coalesce(a, 1) > 0 do nothing; insert into insertconflict values (1, 2) on conflict (b) where coalesce(a, 1) > 1 do nothing; drop table insertconflict; +-- +-- test insertion through view +-- +create table insertconflict (f1 int primary key, f2 text); +create view insertconflictv as + select * from insertconflict with cascaded check option; +insert into insertconflictv values (1,'foo') + on conflict (f1) do update set f2 = excluded.f2; +select * from insertconflict; + f1 | f2 +----+----- + 1 | foo +(1 row) + +insert into insertconflictv values (1,'bar') + on conflict (f1) do update set f2 = excluded.f2; +select * from insertconflict; + f1 | f2 +----+----- + 1 | bar +(1 row) + +drop view insertconflictv; +drop table insertconflict; -- ****************************************************************** -- * * -- * Test inheritance (example taken from tutorial) * diff --git a/src/test/regress/sql/insert_conflict.sql b/src/test/regress/sql/insert_conflict.sql index 81c4a7ca4b..116cf763f9 100644 --- a/src/test/regress/sql/insert_conflict.sql +++ b/src/test/regress/sql/insert_conflict.sql @@ -283,6 +283,24 @@ on conflict (b) where coalesce(a, 1) > 1 do nothing; drop table insertconflict; +-- +-- test insertion through view +-- + +create table insertconflict (f1 int primary key, f2 text); +create view insertconflictv as + select * from insertconflict with cascaded check option; + +insert into insertconflictv values (1,'foo') + on conflict (f1) do update set f2 = excluded.f2; +select * from insertconflict; +insert into insertconflictv values (1,'bar') + on conflict (f1) do update set f2 = excluded.f2; +select * from insertconflict; + +drop view insertconflictv; +drop table insertconflict; + -- ****************************************************************** -- * * From 650b96707672599e290c982dd63e3a896dbbaba6 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Mon, 7 Nov 2016 20:27:30 -0500 Subject: [PATCH 456/871] Change qr/foo$/m to qr/foo\n/m, for Perl 5.8.8. In each case, absence of a trailing newline would itself constitute a PostgreSQL bug. Therefore, this slightly enhances the changed tests. This works around a bug that last appeared in Perl 5.8.8, fixing src/test/modules/test_pg_dump when run against that version. Commit e7293e3271bf618eeb2d4779a15fc516a69fe463 worked around the bug, but the subsequent addition of test_pg_dump introduced affected code. As that commit had shown, slight increases in pattern complexity can suppress the bug. This commit edits qr/foo$/m patterns too complex to encounter the bug today, for style consistency and robustness against unrelated pattern changes. Back-patch to 9.6, where test_pg_dump was introduced. As of this writing, a fresh MSYS installation includes an affected Perl 5.8.8. The Perl 5.8.8 in Red Hat Enterprise Linux 5.11 carries a patch that renders it unaffected, but the Perl 5.8.5 of Red Hat Enterprise Linux 4.4 is affected. --- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 8 ++-- src/test/modules/test_pg_dump/t/001_base.pl | 48 ++++++++++---------- src/test/perl/README | 4 ++ src/tools/msvc/Project.pm | 2 +- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 91eb84e238..7811093100 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -215,16 +215,14 @@ ok(-f "$tempdir/backupR/recovery.conf", 'recovery.conf was created'); my $recovery_conf = slurp_file "$tempdir/backupR/recovery.conf"; -# using a character class for the final "'" here works around an apparent -# bug in several version of the Msys DTK perl my $port = $node->port; like( $recovery_conf, - qr/^standby_mode = 'on[']$/m, + qr/^standby_mode = 'on'\n/m, 'recovery.conf sets standby_mode'); like( $recovery_conf, - qr/^primary_conninfo = '.*port=$port.*'$/m, + qr/^primary_conninfo = '.*port=$port.*'\n/m, 'recovery.conf sets primary_conninfo'); $node->command_ok( @@ -273,5 +271,5 @@ 'pg_basebackup with replication slot and -R runs'); like( slurp_file("$tempdir/backupxs_sl_R/recovery.conf"), - qr/^primary_slot_name = 'slot1'$/m, + qr/^primary_slot_name = 'slot1'\n/m, 'recovery.conf sets primary_slot_name'); diff --git a/src/test/modules/test_pg_dump/t/001_base.pl b/src/test/modules/test_pg_dump/t/001_base.pl index 55f0eb4547..dc90a4aa12 100644 --- a/src/test/modules/test_pg_dump/t/001_base.pl +++ b/src/test/modules/test_pg_dump/t/001_base.pl @@ -193,7 +193,7 @@ create_sql => 'CREATE EXTENSION test_pg_dump;', regexp => qr/^ \QCREATE EXTENSION IF NOT EXISTS test_pg_dump WITH SCHEMA public;\E - $/xm, + \n/xm, like => { clean => 1, clean_if_exists => 1, @@ -210,7 +210,7 @@ 'CREATE ROLE regress_dump_test_role' => { create_order => 1, create_sql => 'CREATE ROLE regress_dump_test_role;', - regexp => qr/^CREATE ROLE regress_dump_test_role;$/m, + regexp => qr/^CREATE ROLE regress_dump_test_role;\n/m, like => { pg_dumpall_globals => 1, }, unlike => { binary_upgrade => 1, @@ -231,7 +231,7 @@ \n\s+\QNO MINVALUE\E \n\s+\QNO MAXVALUE\E \n\s+\QCACHE 1;\E - $/xm, + \n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -252,7 +252,7 @@ \n\s+\QNO MINVALUE\E \n\s+\QNO MAXVALUE\E \n\s+\QCACHE 1;\E - $/xm, + \n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -270,7 +270,7 @@ \QCREATE TABLE regress_pg_dump_table (\E \n\s+\Qcol1 integer NOT NULL,\E \n\s+\Qcol2 integer\E - \n\);$/xm, + \n\);\n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -286,7 +286,7 @@ 'CREATE ACCESS METHOD regress_test_am' => { regexp => qr/^ \QCREATE ACCESS METHOD regress_test_am TYPE INDEX HANDLER bthandler;\E - $/xm, + \n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -303,7 +303,7 @@ regexp => qr/^ \QCOMMENT ON EXTENSION test_pg_dump \E \QIS 'Test pg_dump with an extension';\E - $/xm, + \n/xm, like => { binary_upgrade => 1, clean => 1, @@ -322,7 +322,7 @@ \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n \QGRANT SELECT ON TABLE regress_pg_dump_table TO regress_dump_test_role;\E\n \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E - $/xms, + \n/xms, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -340,7 +340,7 @@ \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n \QGRANT SELECT(col1) ON TABLE regress_pg_dump_table TO PUBLIC;\E\n \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E - $/xms, + \n/xms, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -359,7 +359,7 @@ TO regress_dump_test_role;', regexp => qr/^ \QGRANT SELECT(col2) ON TABLE regress_pg_dump_table TO regress_dump_test_role;\E - $/xm, + \n/xm, like => { binary_upgrade => 1, clean => 1, @@ -380,7 +380,7 @@ TO regress_dump_test_role;', regexp => qr/^ \QGRANT USAGE ON SEQUENCE regress_pg_dump_table_col1_seq TO regress_dump_test_role;\E - $/xm, + \n/xm, like => { binary_upgrade => 1, clean => 1, @@ -397,7 +397,7 @@ 'GRANT USAGE ON regress_pg_dump_seq TO regress_dump_test_role' => { regexp => qr/^ \QGRANT USAGE ON SEQUENCE regress_pg_dump_seq TO regress_dump_test_role;\E - $/xm, + \n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -416,7 +416,7 @@ FROM PUBLIC;', regexp => qr/^ \QREVOKE SELECT(col1) ON TABLE regress_pg_dump_table FROM PUBLIC;\E - $/xm, + \n/xm, like => { binary_upgrade => 1, clean => 1, @@ -436,7 +436,7 @@ \QCREATE TABLE test_table (\E \n\s+\Qcol1 integer,\E \n\s+\Qcol2 integer\E - \n\);$/xm, + \n\);\n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -454,7 +454,7 @@ \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n \QGRANT SELECT ON TABLE test_table TO regress_dump_test_role;\E\n \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E - $/xms, + \n/xms, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -475,7 +475,7 @@ \n\s+\QNO MINVALUE\E \n\s+\QNO MAXVALUE\E \n\s+\QCACHE 1;\E - $/xm, + \n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -493,7 +493,7 @@ \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n \QGRANT USAGE ON SEQUENCE test_seq TO regress_dump_test_role;\E\n \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E - $/xms, + \n/xms, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -510,7 +510,7 @@ regexp => qr/^ \QCREATE TYPE test_type AS (\E \n\s+\Qcol1 integer\E - \n\);$/xm, + \n\);\n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -528,7 +528,7 @@ \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n \QGRANT ALL ON TYPE test_type TO regress_dump_test_role;\E\n \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E - $/xms, + \n/xms, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -545,7 +545,7 @@ regexp => qr/^ \QCREATE FUNCTION test_func() RETURNS integer\E \n\s+\QLANGUAGE sql\E - $/xm, + \n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -563,7 +563,7 @@ \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n \QGRANT ALL ON FUNCTION test_func() TO regress_dump_test_role;\E\n \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E - $/xms, + \n/xms, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -581,7 +581,7 @@ \QCREATE AGGREGATE test_agg(smallint) (\E \n\s+\QSFUNC = int2_sum,\E \n\s+\QSTYPE = bigint\E - \n\);$/xm, + \n\);\n/xm, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -599,7 +599,7 @@ \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(true);\E\n \QGRANT ALL ON FUNCTION test_agg(smallint) TO regress_dump_test_role;\E\n \QSELECT pg_catalog.binary_upgrade_set_record_init_privs(false);\E - $/xms, + \n/xms, like => { binary_upgrade => 1, }, unlike => { clean => 1, @@ -620,7 +620,7 @@ regexp => qr/^ \QCREATE TABLE external_tab (\E \n\s+\Qcol1 integer\E - \n\);$/xm, + \n\);\n/xm, like => { binary_upgrade => 1, clean => 1, diff --git a/src/test/perl/README b/src/test/perl/README index 36d41203ce..710a0d8bc3 100644 --- a/src/test/perl/README +++ b/src/test/perl/README @@ -46,6 +46,10 @@ against them and evaluate the results. For example: $node->stop('fast'); +Test::More::like entails use of the qr// operator. Avoid Perl 5.8.8 bug +#39185 by not using the "$" regular expression metacharacter in qr// when also +using the "/m" modifier. Instead of "$", use "\n" or "(?=\n|\z)". + Read the Test::More documentation for more on how to write tests: perldoc Test::More diff --git a/src/tools/msvc/Project.pm b/src/tools/msvc/Project.pm index a4eb653c26..faf1a683f6 100644 --- a/src/tools/msvc/Project.pm +++ b/src/tools/msvc/Project.pm @@ -256,7 +256,7 @@ sub AddDir # Match rules that pull in source files from different directories, eg # pgstrcasecmp.c rint.c snprintf.c: % : $(top_srcdir)/src/port/% my $replace_re = - qr{^([^:\n\$]+\.c)\s*:\s*(?:%\s*: )?\$(\([^\)]+\))\/(.*)\/[^\/]+$}m; + qr{^([^:\n\$]+\.c)\s*:\s*(?:%\s*: )?\$(\([^\)]+\))\/(.*)\/[^\/]+\n}m; while ($mf =~ m{$replace_re}m) { my $match = $1; From f0e72a25b05d4c29d0102fa0b892782ff193a00e Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 8 Nov 2016 10:47:52 -0500 Subject: [PATCH 457/871] Improve handling of dead tuples in hash indexes. When squeezing a bucket during vacuum, it's not necessary to retain any tuples already marked as dead, so ignore them when deciding which tuples must be moved in order to empty a bucket page. Similarly, when splitting a bucket, relocating dead tuples to the new bucket is a waste of effort; instead, just ignore them. Amit Kapila, reviewed by me. Testing help provided by Ashutosh Sharma. --- src/backend/access/hash/hashovfl.c | 4 ++++ src/backend/access/hash/hashpage.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/backend/access/hash/hashovfl.c b/src/backend/access/hash/hashovfl.c index db3e268a76..df7af3ec84 100644 --- a/src/backend/access/hash/hashovfl.c +++ b/src/backend/access/hash/hashovfl.c @@ -656,6 +656,10 @@ _hash_squeezebucket(Relation rel, IndexTuple itup; Size itemsz; + /* skip dead tuples */ + if (ItemIdIsDead(PageGetItemId(rpage, roffnum))) + continue; + itup = (IndexTuple) PageGetItem(rpage, PageGetItemId(rpage, roffnum)); itemsz = IndexTupleDSize(*itup); diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c index 178463fcb6..a5e9d176a7 100644 --- a/src/backend/access/hash/hashpage.c +++ b/src/backend/access/hash/hashpage.c @@ -811,6 +811,10 @@ _hash_splitbucket(Relation rel, Size itemsz; Bucket bucket; + /* skip dead tuples */ + if (ItemIdIsDead(PageGetItemId(opage, ooffnum))) + continue; + /* * Fetch the item's hash key (conveniently stored in the item) and * determine which bucket it now belongs in. From 0d4446083df56a36ecec8e8bd321a45ecac7e7c6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Nov 2016 11:35:01 -0500 Subject: [PATCH 458/871] Use heap_modify_tuple not SPI_modifytuple in pl/perl triggers. The code here would need some change anyway given planned change in SPI_modifytuple semantics, since this executes after we've exited the SPI environment. But really it's better to just use heap_modify_tuple. The code's actually shorter this way, and this avoids depending on some rather indirect reasoning about why the temporary arrays can't be overrun. (I think the old code is safe, as long as Perl hashes can't contain duplicate keys; but with this way we don't need that assumption, only the assumption that SPI_fnumber doesn't return an out-of-range attnum.) While at it, normalize use of SPI_fnumber: make error messages distinguish no-such-column from can't-set-system-column, and remove test for deleted column which is going to migrate into SPI_fnumber. --- src/pl/plperl/plperl.c | 56 +++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 87113f0fb1..4d993e7371 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -1670,8 +1670,7 @@ plperl_event_trigger_build_args(FunctionCallInfo fcinfo) return newRV_noinc((SV *) hv); } -/* Set up the new tuple returned from a trigger. */ - +/* Construct the modified new tuple to be returned from a trigger. */ static HeapTuple plperl_modify_tuple(HV *hvTD, TriggerData *tdata, HeapTuple otup) { @@ -1679,14 +1678,11 @@ plperl_modify_tuple(HV *hvTD, TriggerData *tdata, HeapTuple otup) HV *hvNew; HE *he; HeapTuple rtup; - int slotsused; - int *modattrs; - Datum *modvalues; - char *modnulls; - TupleDesc tupdesc; - - tupdesc = tdata->tg_relation->rd_att; + int natts; + Datum *modvalues; + bool *modnulls; + bool *modrepls; svp = hv_fetch_string(hvTD, "new"); if (!svp) @@ -1699,51 +1695,49 @@ plperl_modify_tuple(HV *hvTD, TriggerData *tdata, HeapTuple otup) errmsg("$_TD->{new} is not a hash reference"))); hvNew = (HV *) SvRV(*svp); - modattrs = palloc(tupdesc->natts * sizeof(int)); - modvalues = palloc(tupdesc->natts * sizeof(Datum)); - modnulls = palloc(tupdesc->natts * sizeof(char)); - slotsused = 0; + tupdesc = tdata->tg_relation->rd_att; + natts = tupdesc->natts; + + modvalues = (Datum *) palloc0(natts * sizeof(Datum)); + modnulls = (bool *) palloc0(natts * sizeof(bool)); + modrepls = (bool *) palloc0(natts * sizeof(bool)); hv_iterinit(hvNew); while ((he = hv_iternext(hvNew))) { - bool isnull; char *key = hek2cstr(he); SV *val = HeVAL(he); int attn = SPI_fnumber(tupdesc, key); - if (attn <= 0 || tupdesc->attrs[attn - 1]->attisdropped) + if (attn == SPI_ERROR_NOATTRIBUTE) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("Perl hash contains nonexistent column \"%s\"", key))); + if (attn <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot set system attribute \"%s\"", + key))); - modvalues[slotsused] = plperl_sv_to_datum(val, + modvalues[attn - 1] = plperl_sv_to_datum(val, tupdesc->attrs[attn - 1]->atttypid, tupdesc->attrs[attn - 1]->atttypmod, - NULL, - NULL, - InvalidOid, - &isnull); - - modnulls[slotsused] = isnull ? 'n' : ' '; - modattrs[slotsused] = attn; - slotsused++; + NULL, + NULL, + InvalidOid, + &modnulls[attn - 1]); + modrepls[attn - 1] = true; pfree(key); } hv_iterinit(hvNew); - rtup = SPI_modifytuple(tdata->tg_relation, otup, slotsused, - modattrs, modvalues, modnulls); + rtup = heap_modify_tuple(otup, tupdesc, modvalues, modnulls, modrepls); - pfree(modattrs); pfree(modvalues); pfree(modnulls); - - if (rtup == NULL) - elog(ERROR, "SPI_modifytuple failed: %s", - SPI_result_code_string(SPI_result)); + pfree(modrepls); return rtup; } From de4026c673f195cfdb7aa7cc87cc60e36963f094 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Nov 2016 12:00:24 -0500 Subject: [PATCH 459/871] Use heap_modify_tuple not SPI_modifytuple in pl/python triggers. The code here would need some change anyway given planned change in SPI_modifytuple semantics, since this executes after we've exited the SPI environment. But really it's better to just use heap_modify_tuple. While at it, normalize use of SPI_fnumber: make error messages distinguish no-such-column from can't-set-system-column, and remove test for deleted column which is going to migrate into SPI_fnumber. The lack of a check for system column names is actually a pre-existing bug here, and might even qualify as a security bug except that we don't have any trusted version of plpython. --- src/pl/plpython/plpy_exec.c | 86 ++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 45 deletions(-) diff --git a/src/pl/plpython/plpy_exec.c b/src/pl/plpython/plpy_exec.c index fa583fab16..fa5b25a5fa 100644 --- a/src/pl/plpython/plpy_exec.c +++ b/src/pl/plpython/plpy_exec.c @@ -896,18 +896,13 @@ static HeapTuple PLy_modify_tuple(PLyProcedure *proc, PyObject *pltd, TriggerData *tdata, HeapTuple otup) { + HeapTuple rtup; PyObject *volatile plntup; PyObject *volatile plkeys; PyObject *volatile plval; - HeapTuple rtup; - int natts, - i, - attn, - atti; - int *volatile modattrs; Datum *volatile modvalues; - char *volatile modnulls; - TupleDesc tupdesc; + bool *volatile modnulls; + bool *volatile modrepls; ErrorContextCallback plerrcontext; plerrcontext.callback = plpython_trigger_error_callback; @@ -915,12 +910,16 @@ PLy_modify_tuple(PLyProcedure *proc, PyObject *pltd, TriggerData *tdata, error_context_stack = &plerrcontext; plntup = plkeys = plval = NULL; - modattrs = NULL; modvalues = NULL; modnulls = NULL; + modrepls = NULL; PG_TRY(); { + TupleDesc tupdesc; + int nkeys, + i; + if ((plntup = PyDict_GetItemString(pltd, "new")) == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), @@ -932,18 +931,20 @@ PLy_modify_tuple(PLyProcedure *proc, PyObject *pltd, TriggerData *tdata, errmsg("TD[\"new\"] is not a dictionary"))); plkeys = PyDict_Keys(plntup); - natts = PyList_Size(plkeys); - - modattrs = (int *) palloc(natts * sizeof(int)); - modvalues = (Datum *) palloc(natts * sizeof(Datum)); - modnulls = (char *) palloc(natts * sizeof(char)); + nkeys = PyList_Size(plkeys); tupdesc = tdata->tg_relation->rd_att; - for (i = 0; i < natts; i++) + modvalues = (Datum *) palloc0(tupdesc->natts * sizeof(Datum)); + modnulls = (bool *) palloc0(tupdesc->natts * sizeof(bool)); + modrepls = (bool *) palloc0(tupdesc->natts * sizeof(bool)); + + for (i = 0; i < nkeys; i++) { PyObject *platt; char *plattstr; + int attn; + PLyObToDatum *att; platt = PyList_GetItem(plkeys, i); if (PyString_Check(platt)) @@ -963,7 +964,12 @@ PLy_modify_tuple(PLyProcedure *proc, PyObject *pltd, TriggerData *tdata, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("key \"%s\" found in TD[\"new\"] does not exist as a column in the triggering row", plattstr))); - atti = attn - 1; + if (attn <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot set system attribute \"%s\"", + plattstr))); + att = &proc->result.out.r.atts[attn - 1]; plval = PyDict_GetItem(plntup, platt); if (plval == NULL) @@ -971,41 +977,31 @@ PLy_modify_tuple(PLyProcedure *proc, PyObject *pltd, TriggerData *tdata, Py_INCREF(plval); - modattrs[i] = attn; - - if (tupdesc->attrs[atti]->attisdropped) - { - modvalues[i] = (Datum) 0; - modnulls[i] = 'n'; - } - else if (plval != Py_None) + if (plval != Py_None) { - PLyObToDatum *att = &proc->result.out.r.atts[atti]; - - modvalues[i] = (att->func) (att, - tupdesc->attrs[atti]->atttypmod, - plval, - false); - modnulls[i] = ' '; + modvalues[attn - 1] = + (att->func) (att, + tupdesc->attrs[attn - 1]->atttypmod, + plval, + false); + modnulls[attn - 1] = false; } else { - modvalues[i] = - InputFunctionCall(&proc->result.out.r.atts[atti].typfunc, + modvalues[attn - 1] = + InputFunctionCall(&att->typfunc, NULL, - proc->result.out.r.atts[atti].typioparam, - tupdesc->attrs[atti]->atttypmod); - modnulls[i] = 'n'; + att->typioparam, + tupdesc->attrs[attn - 1]->atttypmod); + modnulls[attn - 1] = true; } + modrepls[attn - 1] = true; Py_DECREF(plval); plval = NULL; } - rtup = SPI_modifytuple(tdata->tg_relation, otup, natts, - modattrs, modvalues, modnulls); - if (rtup == NULL) - elog(ERROR, "SPI_modifytuple failed: error %d", SPI_result); + rtup = heap_modify_tuple(otup, tupdesc, modvalues, modnulls, modrepls); } PG_CATCH(); { @@ -1013,12 +1009,12 @@ PLy_modify_tuple(PLyProcedure *proc, PyObject *pltd, TriggerData *tdata, Py_XDECREF(plkeys); Py_XDECREF(plval); - if (modnulls) - pfree(modnulls); if (modvalues) pfree(modvalues); - if (modattrs) - pfree(modattrs); + if (modnulls) + pfree(modnulls); + if (modrepls) + pfree(modrepls); PG_RE_THROW(); } @@ -1027,9 +1023,9 @@ PLy_modify_tuple(PLyProcedure *proc, PyObject *pltd, TriggerData *tdata, Py_DECREF(plntup); Py_DECREF(plkeys); - pfree(modattrs); pfree(modvalues); pfree(modnulls); + pfree(modrepls); error_context_stack = plerrcontext.previous; From 60379f66c8527a260bb1946f703540728d73932d Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 8 Nov 2016 12:09:18 -0500 Subject: [PATCH 460/871] Fix mistake in XLOG_SEG_SIZE test. The intent of the test is to check whether XLOG_SEG_SIZE is in a particular range, but actually in one case it compares XLOG_BLCKSZ by mistake. Repair. Commit 88e982302684246e8af785e78a467ac37c76dee9 introduced this faulty test. Kuntal Ghosh, reviewed by Michael Paquier. --- src/backend/utils/misc/guc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 65660c1bf7..3c695c1615 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -707,7 +707,7 @@ typedef struct #if XLOG_BLCKSZ < 1024 || XLOG_BLCKSZ > (1024*1024) #error XLOG_BLCKSZ must be between 1KB and 1MB #endif -#if XLOG_SEG_SIZE < (1024*1024) || XLOG_BLCKSZ > (1024*1024*1024) +#if XLOG_SEG_SIZE < (1024*1024) || XLOG_SEG_SIZE > (1024*1024*1024) #error XLOG_SEG_SIZE must be between 1MB and 1GB #endif From 36ac6d0e793087153a452df6502d0ef32a780db6 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 8 Nov 2016 18:34:59 +0100 Subject: [PATCH 461/871] Fix typo --- doc/src/sgml/ref/pg_basebackup.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index e66a7ae8ee..1f15a17d0e 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -88,7 +88,7 @@ PostgreSQL documentation There is no guarantee that all WAL files required for the backup are archived at the end of backup. If you are planning to use the backup for an archive recovery and want to ensure that all required files are available at that moment, - you need to include them into the backup by using -x option. + you need to include them into the backup by using the -x option. From 6d30fb1f75a57d80f80e27770d39d88f8aa32d28 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Nov 2016 13:11:15 -0500 Subject: [PATCH 462/871] Make SPI_fnumber() reject dropped columns. There's basically no scenario where it's sensible for this to match dropped columns, so put a test for dropped-ness into SPI_fnumber() itself, and excise the test from the small number of callers that were paying attention to the case. (Most weren't :-(.) In passing, normalize tests at call sites: always reject attnum <= 0 if we're disallowing system columns. Previously there was a mixture of "< 0" and "<= 0" tests. This makes no practical difference since SPI_fnumber() never returns 0, but I'm feeling pedantic today. Also, in the places that are actually live user-facing code and not legacy cruft, distinguish "column not found" from "can't handle system column". Per discussion with Jim Nasby; thi supersedes his original patch that just changed the behavior at one call site. Discussion: --- contrib/spi/autoinc.c | 2 +- contrib/spi/insert_username.c | 2 +- contrib/spi/moddatetime.c | 4 ++-- contrib/spi/refint.c | 5 +++-- contrib/spi/timetravel.c | 4 ++-- doc/src/sgml/spi.sgml | 2 +- src/backend/executor/spi.c | 3 ++- src/backend/utils/adt/tsvector_op.c | 1 + src/pl/plperl/plperl.c | 7 ++++++- src/pl/tcl/pltcl.c | 11 +---------- src/test/regress/regress.c | 9 +++++---- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/contrib/spi/autoinc.c b/contrib/spi/autoinc.c index 41eae4fdc4..fc657a7c06 100644 --- a/contrib/spi/autoinc.c +++ b/contrib/spi/autoinc.c @@ -71,7 +71,7 @@ autoinc(PG_FUNCTION_ARGS) int32 val; Datum seqname; - if (attnum < 0) + if (attnum <= 0) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_ACTION_EXCEPTION), errmsg("\"%s\" has no attribute \"%s\"", diff --git a/contrib/spi/insert_username.c b/contrib/spi/insert_username.c index 3812525c4c..617c60a81c 100644 --- a/contrib/spi/insert_username.c +++ b/contrib/spi/insert_username.c @@ -67,7 +67,7 @@ insert_username(PG_FUNCTION_ARGS) attnum = SPI_fnumber(tupdesc, args[0]); - if (attnum < 0) + if (attnum <= 0) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_ACTION_EXCEPTION), errmsg("\"%s\" has no attribute \"%s\"", relname, args[0]))); diff --git a/contrib/spi/moddatetime.c b/contrib/spi/moddatetime.c index c6d33b7355..cd700fe6d1 100644 --- a/contrib/spi/moddatetime.c +++ b/contrib/spi/moddatetime.c @@ -84,9 +84,9 @@ moddatetime(PG_FUNCTION_ARGS) /* * This is where we check to see if the field we are supposed to update - * even exists. The above function must return -1 if name not found? + * even exists. */ - if (attnum < 0) + if (attnum <= 0) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_ACTION_EXCEPTION), errmsg("\"%s\" has no attribute \"%s\"", diff --git a/contrib/spi/refint.c b/contrib/spi/refint.c index 01dd717522..78cfedf219 100644 --- a/contrib/spi/refint.c +++ b/contrib/spi/refint.c @@ -135,7 +135,7 @@ check_primary_key(PG_FUNCTION_ARGS) int fnumber = SPI_fnumber(tupdesc, args[i]); /* Bad guys may give us un-existing column in CREATE TRIGGER */ - if (fnumber < 0) + if (fnumber <= 0) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("there is no attribute \"%s\" in relation \"%s\"", @@ -362,7 +362,7 @@ check_foreign_key(PG_FUNCTION_ARGS) int fnumber = SPI_fnumber(tupdesc, args[i]); /* Bad guys may give us un-existing column in CREATE TRIGGER */ - if (fnumber < 0) + if (fnumber <= 0) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("there is no attribute \"%s\" in relation \"%s\"", @@ -469,6 +469,7 @@ check_foreign_key(PG_FUNCTION_ARGS) char *type; fn = SPI_fnumber(tupdesc, args_temp[k - 1]); + Assert(fn > 0); /* already checked above */ nv = SPI_getvalue(newtuple, tupdesc, fn); type = SPI_gettype(tupdesc, fn); diff --git a/contrib/spi/timetravel.c b/contrib/spi/timetravel.c index 5a345841c6..30dcfd4d3e 100644 --- a/contrib/spi/timetravel.c +++ b/contrib/spi/timetravel.c @@ -157,7 +157,7 @@ timetravel(PG_FUNCTION_ARGS) for (i = 0; i < MinAttrNum; i++) { attnum[i] = SPI_fnumber(tupdesc, args[i]); - if (attnum[i] < 0) + if (attnum[i] <= 0) elog(ERROR, "timetravel (%s): there is no attribute %s", relname, args[i]); if (SPI_gettypeid(tupdesc, attnum[i]) != ABSTIMEOID) elog(ERROR, "timetravel (%s): attribute %s must be of abstime type", @@ -166,7 +166,7 @@ timetravel(PG_FUNCTION_ARGS) for (; i < argc; i++) { attnum[i] = SPI_fnumber(tupdesc, args[i]); - if (attnum[i] < 0) + if (attnum[i] <= 0) elog(ERROR, "timetravel (%s): there is no attribute %s", relname, args[i]); if (SPI_gettypeid(tupdesc, attnum[i]) != TEXTOID) elog(ERROR, "timetravel (%s): attribute %s must be of text type", diff --git a/doc/src/sgml/spi.sgml b/doc/src/sgml/spi.sgml index 9ae7126ae7..817a5d0120 100644 --- a/doc/src/sgml/spi.sgml +++ b/doc/src/sgml/spi.sgml @@ -2891,7 +2891,7 @@ int SPI_fnumber(TupleDesc rowdesc, const char * Return Value - Column number (count starts at 1), or + Column number (count starts at 1 for user-defined columns), or SPI_ERROR_NOATTRIBUTE if the named column was not found. diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index 38767ae4ce..8e650bc412 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -824,7 +824,8 @@ SPI_fnumber(TupleDesc tupdesc, const char *fname) for (res = 0; res < tupdesc->natts; res++) { - if (namestrcmp(&tupdesc->attrs[res]->attname, fname) == 0) + if (namestrcmp(&tupdesc->attrs[res]->attname, fname) == 0 && + !tupdesc->attrs[res]->attisdropped) return res + 1; } diff --git a/src/backend/utils/adt/tsvector_op.c b/src/backend/utils/adt/tsvector_op.c index ad5a254c57..0e9ae5ff9c 100644 --- a/src/backend/utils/adt/tsvector_op.c +++ b/src/backend/utils/adt/tsvector_op.c @@ -2242,6 +2242,7 @@ tsvector_update_trigger(PG_FUNCTION_ARGS, bool config_column) (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("tsvector column \"%s\" does not exist", trigger->tgargs[0]))); + /* This will effectively reject system columns, so no separate test: */ if (!IsBinaryCoercible(SPI_gettypeid(rel->rd_att, tsvector_attr_num), TSVECTOROID)) ereport(ERROR, diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 4d993e7371..461986cda3 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -1062,11 +1062,16 @@ plperl_build_tuple_result(HV *perlhash, TupleDesc td) char *key = hek2cstr(he); int attn = SPI_fnumber(td, key); - if (attn <= 0 || td->attrs[attn - 1]->attisdropped) + if (attn == SPI_ERROR_NOATTRIBUTE) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("Perl hash contains nonexistent column \"%s\"", key))); + if (attn <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot set system attribute \"%s\"", + key))); values[attn - 1] = plperl_sv_to_datum(val, td->attrs[attn - 1]->atttypid, diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 3e52113ee2..20809102ef 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -603,6 +603,7 @@ pltcl_init_load_unknown(Tcl_Interp *interp) * leave this code as DString - it's only executed once per session ************************************************************/ fno = SPI_fnumber(SPI_tuptable->tupdesc, "modsrc"); + Assert(fno > 0); Tcl_DStringInit(&unknown_src); @@ -1259,12 +1260,6 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, errmsg("cannot set system attribute \"%s\"", ret_name))); - /************************************************************ - * Ignore dropped columns - ************************************************************/ - if (tupdesc->attrs[attnum - 1]->attisdropped) - continue; - /************************************************************ * Lookup the attribute type's input function ************************************************************/ @@ -3077,10 +3072,6 @@ pltcl_build_tuple_result(Tcl_Interp *interp, Tcl_Obj **kvObjv, int kvObjc, errmsg("cannot set system attribute \"%s\"", fieldName))); - /* Ignore dropped attributes */ - if (call_state->ret_tupdesc->attrs[attn - 1]->attisdropped) - continue; - values[attn - 1] = utf_e2u(Tcl_GetString(kvObjv[i + 1])); } diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index e7826a4513..119a59ab07 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -523,11 +523,12 @@ ttdummy(PG_FUNCTION_ARGS) for (i = 0; i < 2; i++) { attnum[i] = SPI_fnumber(tupdesc, args[i]); - if (attnum[i] < 0) - elog(ERROR, "ttdummy (%s): there is no attribute %s", relname, args[i]); + if (attnum[i] <= 0) + elog(ERROR, "ttdummy (%s): there is no attribute %s", + relname, args[i]); if (SPI_gettypeid(tupdesc, attnum[i]) != INT4OID) - elog(ERROR, "ttdummy (%s): attributes %s and %s must be of abstime type", - relname, args[0], args[1]); + elog(ERROR, "ttdummy (%s): attribute %s must be of integer type", + relname, args[i]); } oldon = SPI_getbinval(trigtuple, tupdesc, attnum[0], &isnull); From dce429b117be027f059bb9df5c76eb5eadcc456d Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 8 Nov 2016 15:33:57 -0500 Subject: [PATCH 463/871] Fix typo. Michael Paquier --- src/backend/commands/event_trigger.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index ac4c4ecbe7..e87fce7e6a 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -742,7 +742,7 @@ EventTriggerCommonSetup(Node *parsetree, /* * Filter list of event triggers by command tag, and copy them into our - * memory context. Once we start running the command trigers, or indeed + * memory context. Once we start running the command triggers, or indeed * once we do anything at all that touches the catalogs, an invalidation * might leave cachelist pointing at garbage, so we must do this before we * can do much else. From 9257f0787257022e31c61cd77449127adfccf37f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Nov 2016 15:36:36 -0500 Subject: [PATCH 464/871] Replace uses of SPI_modifytuple that intend to allocate in current context. Invent a new function heap_modify_tuple_by_cols() that is functionally equivalent to SPI_modifytuple except that it always allocates its result by simple palloc. I chose however to make the API details a bit more like heap_modify_tuple: pass a tupdesc rather than a Relation, and use bool convention for the isnull array. Use this function in place of SPI_modifytuple at all call sites where the intended behavior is to allocate in current context. (There actually are only two call sites left that depend on the old behavior, which makes me wonder if we should just drop this function rather than keep it.) This new function is easier to use than heap_modify_tuple() for purposes of replacing a single column (or, really, any fixed number of columns). There are a number of places where it would simplify the code to change over, but I resisted that temptation for the moment ... everywhere except in plpgsql's exec_assign_value(); changing that might offer some small performance benefit, so I did it. This is on the way to removing SPI_push/SPI_pop, but it seems like good code cleanup in its own right. Discussion: <9633.1478552022@sss.pgh.pa.us> --- contrib/spi/autoinc.c | 13 ++++-- contrib/spi/insert_username.c | 12 ++--- contrib/spi/moddatetime.c | 21 +++------ contrib/spi/timetravel.c | 25 +++++----- doc/src/sgml/spi.sgml | 5 +- src/backend/access/common/heaptuple.c | 66 +++++++++++++++++++++++++++ src/backend/utils/adt/tsvector_op.c | 16 +++---- src/include/access/htup_details.h | 6 +++ src/pl/plpgsql/src/pl_exec.c | 57 ++++++++--------------- src/test/regress/regress.c | 11 +---- 10 files changed, 137 insertions(+), 95 deletions(-) diff --git a/contrib/spi/autoinc.c b/contrib/spi/autoinc.c index fc657a7c06..54f85a3709 100644 --- a/contrib/spi/autoinc.c +++ b/contrib/spi/autoinc.c @@ -3,6 +3,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" @@ -23,6 +24,7 @@ autoinc(PG_FUNCTION_ARGS) int *chattrs; /* attnums of attributes to change */ int chnattrs = 0; /* # of above */ Datum *newvals; /* vals of above */ + bool *newnulls; /* null flags for above */ char **args; /* arguments */ char *relname; /* triggered relation name */ Relation rel; /* triggered relation */ @@ -64,6 +66,7 @@ autoinc(PG_FUNCTION_ARGS) chattrs = (int *) palloc(nargs / 2 * sizeof(int)); newvals = (Datum *) palloc(nargs / 2 * sizeof(Datum)); + newnulls = (bool *) palloc(nargs / 2 * sizeof(bool)); for (i = 0; i < nargs;) { @@ -102,6 +105,7 @@ autoinc(PG_FUNCTION_ARGS) newvals[chnattrs] = DirectFunctionCall1(nextval, seqname); newvals[chnattrs] = Int32GetDatum((int32) DatumGetInt64(newvals[chnattrs])); } + newnulls[chnattrs] = false; pfree(DatumGetTextP(seqname)); chnattrs++; i++; @@ -109,16 +113,15 @@ autoinc(PG_FUNCTION_ARGS) if (chnattrs > 0) { - rettuple = SPI_modifytuple(rel, rettuple, chnattrs, chattrs, newvals, NULL); - if (rettuple == NULL) - /* internal error */ - elog(ERROR, "autoinc (%s): %d returned by SPI_modifytuple", - relname, SPI_result); + rettuple = heap_modify_tuple_by_cols(rettuple, tupdesc, + chnattrs, chattrs, + newvals, newnulls); } pfree(relname); pfree(chattrs); pfree(newvals); + pfree(newnulls); return PointerGetDatum(rettuple); } diff --git a/contrib/spi/insert_username.c b/contrib/spi/insert_username.c index 617c60a81c..a2e1747ff7 100644 --- a/contrib/spi/insert_username.c +++ b/contrib/spi/insert_username.c @@ -1,6 +1,4 @@ /* - * insert_username.c - * $Modified: Thu Oct 16 08:13:42 1997 by brook $ * contrib/spi/insert_username.c * * insert user name in response to a trigger @@ -8,6 +6,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "catalog/pg_type.h" #include "commands/trigger.h" #include "executor/spi.h" @@ -26,6 +25,7 @@ insert_username(PG_FUNCTION_ARGS) Trigger *trigger; /* to get trigger name */ int nargs; /* # of arguments */ Datum newval; /* new value of column */ + bool newnull; /* null flag */ char **args; /* arguments */ char *relname; /* triggered relation name */ Relation rel; /* triggered relation */ @@ -80,13 +80,11 @@ insert_username(PG_FUNCTION_ARGS) /* create fields containing name */ newval = CStringGetTextDatum(GetUserNameFromId(GetUserId(), false)); + newnull = false; /* construct new tuple */ - rettuple = SPI_modifytuple(rel, rettuple, 1, &attnum, &newval, NULL); - if (rettuple == NULL) - /* internal error */ - elog(ERROR, "insert_username (\"%s\"): %d returned by SPI_modifytuple", - relname, SPI_result); + rettuple = heap_modify_tuple_by_cols(rettuple, tupdesc, + 1, &attnum, &newval, &newnull); pfree(relname); diff --git a/contrib/spi/moddatetime.c b/contrib/spi/moddatetime.c index cd700fe6d1..2d1f22c4e1 100644 --- a/contrib/spi/moddatetime.c +++ b/contrib/spi/moddatetime.c @@ -15,6 +15,7 @@ OH, me, I'm Terry Mackintosh */ #include "postgres.h" +#include "access/htup_details.h" #include "catalog/pg_type.h" #include "executor/spi.h" #include "commands/trigger.h" @@ -34,6 +35,7 @@ moddatetime(PG_FUNCTION_ARGS) int attnum; /* positional number of field to change */ Oid atttypid; /* type OID of field to change */ Datum newdt; /* The current datetime. */ + bool newdtnull; /* null flag for it */ char **args; /* arguments */ char *relname; /* triggered relation name */ Relation rel; /* triggered relation */ @@ -115,22 +117,13 @@ moddatetime(PG_FUNCTION_ARGS) args[0], relname))); newdt = (Datum) 0; /* keep compiler quiet */ } + newdtnull = false; -/* 1 is the number of items in the arrays attnum and newdt. - attnum is the positional number of the field to be updated. - newdt is the new datetime stamp. - NOTE that attnum and newdt are not arrays, but then a 1 element array - is not an array any more then they are. Thus, they can be considered a - one element array. -*/ - rettuple = SPI_modifytuple(rel, rettuple, 1, &attnum, &newdt, NULL); - - if (rettuple == NULL) - /* internal error */ - elog(ERROR, "moddatetime (%s): %d returned by SPI_modifytuple", - relname, SPI_result); + /* Replace the attnum'th column with newdt */ + rettuple = heap_modify_tuple_by_cols(rettuple, tupdesc, + 1, &attnum, &newdt, &newdtnull); -/* Clean up */ + /* Clean up */ pfree(relname); return PointerGetDatum(rettuple); diff --git a/contrib/spi/timetravel.c b/contrib/spi/timetravel.c index 30dcfd4d3e..2733aa231e 100644 --- a/contrib/spi/timetravel.c +++ b/contrib/spi/timetravel.c @@ -11,6 +11,7 @@ #include +#include "access/htup_details.h" #include "catalog/pg_type.h" #include "commands/trigger.h" #include "executor/spi.h" @@ -183,13 +184,13 @@ timetravel(PG_FUNCTION_ARGS) int chnattrs = 0; int chattrs[MaxAttrNum]; Datum newvals[MaxAttrNum]; - char newnulls[MaxAttrNum]; + bool newnulls[MaxAttrNum]; oldtimeon = SPI_getbinval(trigtuple, tupdesc, attnum[a_time_on], &isnull); if (isnull) { newvals[chnattrs] = GetCurrentAbsoluteTime(); - newnulls[chnattrs] = ' '; + newnulls[chnattrs] = false; chattrs[chnattrs] = attnum[a_time_on]; chnattrs++; } @@ -201,7 +202,7 @@ timetravel(PG_FUNCTION_ARGS) (chnattrs > 0 && DatumGetInt32(newvals[a_time_on]) >= NOEND_ABSTIME)) elog(ERROR, "timetravel (%s): %s is infinity", relname, args[a_time_on]); newvals[chnattrs] = NOEND_ABSTIME; - newnulls[chnattrs] = ' '; + newnulls[chnattrs] = false; chattrs[chnattrs] = attnum[a_time_off]; chnattrs++; } @@ -220,21 +221,23 @@ timetravel(PG_FUNCTION_ARGS) { /* clear update_user value */ newvals[chnattrs] = nulltext; - newnulls[chnattrs] = 'n'; + newnulls[chnattrs] = true; chattrs[chnattrs] = attnum[a_upd_user]; chnattrs++; /* clear delete_user value */ newvals[chnattrs] = nulltext; - newnulls[chnattrs] = 'n'; + newnulls[chnattrs] = true; chattrs[chnattrs] = attnum[a_del_user]; chnattrs++; /* set insert_user value */ newvals[chnattrs] = newuser; - newnulls[chnattrs] = ' '; + newnulls[chnattrs] = false; chattrs[chnattrs] = attnum[a_ins_user]; chnattrs++; } - rettuple = SPI_modifytuple(rel, trigtuple, chnattrs, chattrs, newvals, newnulls); + rettuple = heap_modify_tuple_by_cols(trigtuple, tupdesc, + chnattrs, chattrs, + newvals, newnulls); return PointerGetDatum(rettuple); /* end of INSERT */ } @@ -395,13 +398,11 @@ timetravel(PG_FUNCTION_ARGS) chnattrs++; } - rettuple = SPI_modifytuple(rel, newtuple, chnattrs, chattrs, newvals, newnulls); - /* - * SPI_copytuple allocates tmptuple in upper executor context - have - * to free allocation using SPI_pfree + * Use SPI_modifytuple() here because we are inside SPI environment + * but rettuple must be allocated in caller's context. */ - /* SPI_pfree(tmptuple); */ + rettuple = SPI_modifytuple(rel, newtuple, chnattrs, chattrs, newvals, newnulls); } else /* DELETE case */ diff --git a/doc/src/sgml/spi.sgml b/doc/src/sgml/spi.sgml index 817a5d0120..39133c9038 100644 --- a/doc/src/sgml/spi.sgml +++ b/doc/src/sgml/spi.sgml @@ -3382,8 +3382,9 @@ char * SPI_getnspname(Relation rel) repalloc, or SPI utility functions (except for SPI_copytuple, SPI_returntuple, - SPI_modifytuple, and - SPI_palloc) are made in this context. When a + SPI_modifytuple, + SPI_palloc, and + SPI_datumTransfer) are made in this context. When a procedure disconnects from the SPI manager (via SPI_finish) the current context is restored to the upper executor context, and all allocations made in the diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index 6d0f3f3767..e27ec78b71 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -846,6 +846,72 @@ heap_modify_tuple(HeapTuple tuple, return newTuple; } +/* + * heap_modify_tuple_by_cols + * form a new tuple from an old tuple and a set of replacement values. + * + * This is like heap_modify_tuple, except that instead of specifying which + * column(s) to replace by a boolean map, an array of target column numbers + * is used. This is often more convenient when a fixed number of columns + * are to be replaced. The replCols, replValues, and replIsnull arrays must + * be of length nCols. Target column numbers are indexed from 1. + * + * The result is allocated in the current memory context. + */ +HeapTuple +heap_modify_tuple_by_cols(HeapTuple tuple, + TupleDesc tupleDesc, + int nCols, + int *replCols, + Datum *replValues, + bool *replIsnull) +{ + int numberOfAttributes = tupleDesc->natts; + Datum *values; + bool *isnull; + HeapTuple newTuple; + int i; + + /* + * allocate and fill values and isnull arrays from the tuple, then replace + * selected columns from the input arrays. + */ + values = (Datum *) palloc(numberOfAttributes * sizeof(Datum)); + isnull = (bool *) palloc(numberOfAttributes * sizeof(bool)); + + heap_deform_tuple(tuple, tupleDesc, values, isnull); + + for (i = 0; i < nCols; i++) + { + int attnum = replCols[i]; + + if (attnum <= 0 || attnum > numberOfAttributes) + elog(ERROR, "invalid column number %d", attnum); + values[attnum - 1] = replValues[i]; + isnull[attnum - 1] = replIsnull[i]; + } + + /* + * create a new tuple from the values and isnull arrays + */ + newTuple = heap_form_tuple(tupleDesc, values, isnull); + + pfree(values); + pfree(isnull); + + /* + * copy the identification info of the old tuple: t_ctid, t_self, and OID + * (if any) + */ + newTuple->t_data->t_ctid = tuple->t_data->t_ctid; + newTuple->t_self = tuple->t_self; + newTuple->t_tableOid = tuple->t_tableOid; + if (tupleDesc->tdhasoid) + HeapTupleSetOid(newTuple, HeapTupleGetOid(tuple)); + + return newTuple; +} + /* * heap_deform_tuple * Given a tuple, extract data into values/isnull arrays; this is diff --git a/src/backend/utils/adt/tsvector_op.c b/src/backend/utils/adt/tsvector_op.c index 0e9ae5ff9c..c9d5060f2c 100644 --- a/src/backend/utils/adt/tsvector_op.c +++ b/src/backend/utils/adt/tsvector_op.c @@ -2329,8 +2329,10 @@ tsvector_update_trigger(PG_FUNCTION_ARGS, bool config_column) if (prs.curwords) { datum = PointerGetDatum(make_tsvector(&prs)); - rettuple = SPI_modifytuple(rel, rettuple, 1, &tsvector_attr_num, - &datum, NULL); + isnull = false; + rettuple = heap_modify_tuple_by_cols(rettuple, rel->rd_att, + 1, &tsvector_attr_num, + &datum, &isnull); pfree(DatumGetPointer(datum)); } else @@ -2340,14 +2342,12 @@ tsvector_update_trigger(PG_FUNCTION_ARGS, bool config_column) SET_VARSIZE(out, CALCDATASIZE(0, 0)); out->size = 0; datum = PointerGetDatum(out); - rettuple = SPI_modifytuple(rel, rettuple, 1, &tsvector_attr_num, - &datum, NULL); + isnull = false; + rettuple = heap_modify_tuple_by_cols(rettuple, rel->rd_att, + 1, &tsvector_attr_num, + &datum, &isnull); pfree(prs.words); } - if (rettuple == NULL) /* internal error */ - elog(ERROR, "tsvector_update_trigger: %d returned by SPI_modifytuple", - SPI_result); - return PointerGetDatum(rettuple); } diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index d7e5fad11e..8fb1f6ddea 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -805,6 +805,12 @@ extern HeapTuple heap_modify_tuple(HeapTuple tuple, Datum *replValues, bool *replIsnull, bool *doReplace); +extern HeapTuple heap_modify_tuple_by_cols(HeapTuple tuple, + TupleDesc tupleDesc, + int nCols, + int *replCols, + Datum *replValues, + bool *replIsnull); extern void heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *values, bool *isnull); extern void heap_freetuple(HeapTuple htup); diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index 042b31fd77..91e1f8dd3f 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -4562,10 +4562,9 @@ exec_assign_value(PLpgSQL_execstate *estate, PLpgSQL_rec *rec; int fno; HeapTuple newtup; - int natts; - Datum *values; - bool *nulls; - bool *replaces; + int colnums[1]; + Datum values[1]; + bool nulls[1]; Oid atttype; int32 atttypmod; @@ -4584,9 +4583,8 @@ exec_assign_value(PLpgSQL_execstate *estate, errdetail("The tuple structure of a not-yet-assigned record is indeterminate."))); /* - * Get the number of the records field to change and the - * number of attributes in the tuple. Note: disallow system - * column names because the code below won't cope. + * Get the number of the record field to change. Disallow + * system columns because the code below won't cope. */ fno = SPI_fnumber(rec->tupdesc, recfield->fieldname); if (fno <= 0) @@ -4594,42 +4592,25 @@ exec_assign_value(PLpgSQL_execstate *estate, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("record \"%s\" has no field \"%s\"", rec->refname, recfield->fieldname))); - fno--; - natts = rec->tupdesc->natts; - - /* - * Set up values/control arrays for heap_modify_tuple. For all - * the attributes except the one we want to replace, use the - * value that's in the old tuple. - */ - values = eval_mcontext_alloc(estate, sizeof(Datum) * natts); - nulls = eval_mcontext_alloc(estate, sizeof(bool) * natts); - replaces = eval_mcontext_alloc(estate, sizeof(bool) * natts); - - memset(replaces, false, sizeof(bool) * natts); - replaces[fno] = true; + colnums[0] = fno; /* * Now insert the new value, being careful to cast it to the * right type. */ - atttype = rec->tupdesc->attrs[fno]->atttypid; - atttypmod = rec->tupdesc->attrs[fno]->atttypmod; - values[fno] = exec_cast_value(estate, - value, - &isNull, - valtype, - valtypmod, - atttype, - atttypmod); - nulls[fno] = isNull; - - /* - * Now call heap_modify_tuple() to create a new tuple that - * replaces the old one in the record. - */ - newtup = heap_modify_tuple(rec->tup, rec->tupdesc, - values, nulls, replaces); + atttype = rec->tupdesc->attrs[fno - 1]->atttypid; + atttypmod = rec->tupdesc->attrs[fno - 1]->atttypmod; + values[0] = exec_cast_value(estate, + value, + &isNull, + valtype, + valtypmod, + atttype, + atttypmod); + nulls[0] = isNull; + + newtup = heap_modify_tuple_by_cols(rec->tup, rec->tupdesc, + 1, colnums, values, nulls); if (rec->freetup) heap_freetuple(rec->tup); diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 119a59ab07..32703fcdcf 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -639,15 +639,8 @@ ttdummy(PG_FUNCTION_ARGS) /* Tuple to return to upper Executor ... */ if (newtuple) /* UPDATE */ - { - HeapTuple tmptuple; - - tmptuple = SPI_copytuple(trigtuple); - rettuple = SPI_modifytuple(rel, tmptuple, 1, &(attnum[1]), &newoff, NULL); - SPI_freetuple(tmptuple); - } - else - /* DELETE */ + rettuple = SPI_modifytuple(rel, trigtuple, 1, &(attnum[1]), &newoff, NULL); + else /* DELETE */ rettuple = trigtuple; SPI_finish(); /* don't forget say Bye to SPI mgr */ From 577f0bdd2b8904cbdfde6c98f4bda6fd93a05ffc Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 8 Nov 2016 16:27:09 -0500 Subject: [PATCH 465/871] psql: Tab completion for renaming enum values. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For ALTER TYPE .. RENAME, add "VALUE" to the list of possible completions. Complete ALTER TYPE .. RENAME VALUE with possible enum values. After that, complete with "TO". Dagfinn Ilmari Mannsåker, reviewed by Artur Zakirov. --- src/bin/psql/tab-complete.c | 58 ++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index a43bbc519c..b556c00b31 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -202,6 +202,31 @@ do { \ matches = completion_matches(text, complete_from_query); \ } while (0) +#define COMPLETE_WITH_ENUM_VALUE(type) \ +do { \ + char *_completion_schema; \ + char *_completion_type; \ +\ + _completion_schema = strtokx(type, " \t\n\r", ".", "\"", 0, \ + false, false, pset.encoding); \ + (void) strtokx(NULL, " \t\n\r", ".", "\"", 0, \ + false, false, pset.encoding); \ + _completion_type = strtokx(NULL, " \t\n\r", ".", "\"", 0, \ + false, false, pset.encoding); \ + if (_completion_type == NULL)\ + { \ + completion_charp = Query_for_list_of_enum_values; \ + completion_info_charp = type; \ + } \ + else \ + { \ + completion_charp = Query_for_list_of_enum_values_with_schema; \ + completion_info_charp = _completion_type; \ + completion_info_charp2 = _completion_schema; \ + } \ + matches = completion_matches(text, complete_from_query); \ +} while (0) + #define COMPLETE_WITH_FUNCTION_ARG(function) \ do { \ char *_completion_schema; \ @@ -598,6 +623,26 @@ static const SchemaQuery Query_for_list_of_matviews = { " AND (pg_catalog.quote_ident(nspname)='%s' "\ " OR '\"' || nspname || '\"' ='%s') " +#define Query_for_list_of_enum_values \ +"SELECT pg_catalog.quote_literal(enumlabel) "\ +" FROM pg_catalog.pg_enum e, pg_catalog.pg_type t "\ +" WHERE t.oid = e.enumtypid "\ +" AND substring(pg_catalog.quote_literal(enumlabel),1,%d)='%s' "\ +" AND (pg_catalog.quote_ident(typname)='%s' "\ +" OR '\"' || typname || '\"'='%s') "\ +" AND pg_catalog.pg_type_is_visible(t.oid)" + +#define Query_for_list_of_enum_values_with_schema \ +"SELECT pg_catalog.quote_literal(enumlabel) "\ +" FROM pg_catalog.pg_enum e, pg_catalog.pg_type t, pg_catalog.pg_namespace n "\ +" WHERE t.oid = e.enumtypid "\ +" AND n.oid = t.typnamespace "\ +" AND substring(pg_catalog.quote_literal(enumlabel),1,%d)='%s' "\ +" AND (pg_catalog.quote_ident(typname)='%s' "\ +" OR '\"' || typname || '\"'='%s') "\ +" AND (pg_catalog.quote_ident(nspname)='%s' "\ +" OR '\"' || nspname || '\"' ='%s') " + #define Query_for_list_of_template_databases \ "SELECT pg_catalog.quote_ident(d.datname) "\ " FROM pg_catalog.pg_database d "\ @@ -1872,11 +1917,10 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH_LIST2("ATTRIBUTE", "VALUE"); /* ALTER TYPE RENAME */ else if (Matches4("ALTER", "TYPE", MatchAny, "RENAME")) - COMPLETE_WITH_LIST2("ATTRIBUTE", "TO"); - /* ALTER TYPE xxx RENAME ATTRIBUTE yyy */ - else if (Matches6("ALTER", "TYPE", MatchAny, "RENAME", "ATTRIBUTE", MatchAny)) + COMPLETE_WITH_LIST3("ATTRIBUTE", "TO", "VALUE"); + /* ALTER TYPE xxx RENAME (ATTRIBUTE|VALUE) yyy */ + else if (Matches6("ALTER", "TYPE", MatchAny, "RENAME", "ATTRIBUTE|VALUE", MatchAny)) COMPLETE_WITH_CONST("TO"); - /* * If we have ALTER TYPE ALTER/DROP/RENAME ATTRIBUTE, provide list * of attributes @@ -1896,6 +1940,12 @@ psql_completion(const char *text, int start, int end) else if (Matches5("ALTER", "GROUP", MatchAny, "ADD|DROP", "USER")) COMPLETE_WITH_QUERY(Query_for_list_of_roles); + /* + * If we have ALTER TYPE RENAME VALUE, provide list of enum values + */ + else if (Matches5("ALTER", "TYPE", MatchAny, "RENAME", "VALUE")) + COMPLETE_WITH_ENUM_VALUE(prev3_wd); + /* BEGIN */ else if (Matches1("BEGIN")) COMPLETE_WITH_LIST6("WORK", "TRANSACTION", "ISOLATION LEVEL", "READ", "DEFERRABLE", "NOT DEFERRABLE"); From 1833f1a1c3b0e12b3ea40d49bf11898eedae5248 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Nov 2016 17:39:45 -0500 Subject: [PATCH 466/871] Simplify code by getting rid of SPI_push, SPI_pop, SPI_restore_connection. The idea behind SPI_push was to allow transitioning back into an "unconnected" state when a SPI-using procedure calls unrelated code that might or might not invoke SPI. That sounds good, but in practice the only thing it does for us is to catch cases where a called SPI-using function forgets to call SPI_connect --- which is a highly improbable failure mode, since it would be exposed immediately by direct testing of said function. As against that, we've had multiple bugs induced by forgetting to call SPI_push/SPI_pop around code that might invoke SPI-using functions; these are much harder to catch and indeed have gone undetected for years in some cases. And we've had to band-aid around some problems of this ilk by introducing conditional push/pop pairs in some places, which really kind of defeats the purpose altogether; if we can't draw bright lines between connected and unconnected code, what's the point? Hence, get rid of SPI_push[_conditional], SPI_pop[_conditional], and the underlying state variable _SPI_curid. It turns out SPI_restore_connection can go away too, which is a nice side benefit since it was never more than a kluge. Provide no-op macros for the deleted functions so as to avoid an API break for external modules. A side effect of this removal is that SPI_palloc and allied functions no longer permit being called when unconnected; they'll throw an error instead. The apparent usefulness of the previous behavior was a mirage as well, because it was depended on by only a few places (which I fixed in preceding commits), and it posed a risk of allocations being unexpectedly long-lived if someone forgot a SPI_push call. Discussion: <20808.1478481403@sss.pgh.pa.us> --- doc/src/sgml/spi.sgml | 180 ++++++------------------ src/backend/executor/spi.c | 200 +++++++-------------------- src/backend/utils/adt/xml.c | 6 - src/backend/utils/cache/plancache.c | 13 -- src/backend/utils/fmgr/fmgr.c | 48 +------ src/include/executor/spi.h | 12 +- src/pl/plperl/plperl.c | 78 ----------- src/pl/plpgsql/src/pl_exec.c | 21 --- src/pl/plpython/plpy_exec.c | 2 - src/pl/plpython/plpy_spi.c | 13 -- src/pl/plpython/plpy_subxactobject.c | 7 - src/pl/tcl/pltcl.c | 18 --- 12 files changed, 105 insertions(+), 493 deletions(-) diff --git a/doc/src/sgml/spi.sgml b/doc/src/sgml/spi.sgml index 39133c9038..836ce0822f 100644 --- a/doc/src/sgml/spi.sgml +++ b/doc/src/sgml/spi.sgml @@ -90,21 +90,6 @@ int SPI_connect(void) function if you want to execute commands through SPI. Some utility SPI functions can be called from unconnected procedures. - - - If your procedure is already connected, - SPI_connect will return the error code - SPI_ERROR_CONNECT. This could happen if - a procedure that has called SPI_connect - directly calls another procedure that calls - SPI_connect. While recursive calls to the - SPI manager are permitted when an SQL command - called through SPI invokes another function that uses - SPI, directly nested calls to - SPI_connect and - SPI_finish are forbidden. - (But see SPI_push and SPI_pop.) - @@ -164,13 +149,6 @@ int SPI_finish(void) abort the transaction via elog(ERROR). In that case SPI will clean itself up automatically. - - - If SPI_finish is called without having a valid - connection, it will return SPI_ERROR_UNCONNECTED. - There is no fundamental problem with this; it means that the SPI - manager has nothing to do. - @@ -200,86 +178,6 @@ int SPI_finish(void) - - SPI_push - - - SPI_push - 3 - - - - SPI_push - push SPI stack to allow recursive SPI usage - - - - -void SPI_push(void) - - - - - Description - - - SPI_push should be called before executing another - procedure that might itself wish to use SPI. - After SPI_push, SPI is no longer in a - connected state, and SPI function calls will be rejected unless - a fresh SPI_connect is done. This ensures a clean - separation between your procedure's SPI state and that of another procedure - you call. After the other procedure returns, call - SPI_pop to restore access to your own SPI state. - - - - Note that SPI_execute and related functions - automatically do the equivalent of SPI_push before - passing control back to the SQL execution engine, so it is not necessary - for you to worry about this when using those functions. - Only when you are directly calling arbitrary code that might contain - SPI_connect calls do you need to issue - SPI_push and SPI_pop. - - - - - - - - - SPI_pop - - - SPI_pop - 3 - - - - SPI_pop - pop SPI stack to return from recursive SPI usage - - - - -void SPI_pop(void) - - - - - Description - - - SPI_pop pops the previous environment from the - SPI call stack. See SPI_push. - - - - - - - SPI_execute @@ -3361,17 +3259,8 @@ char * SPI_getnspname(Relation rel) upper executor context, that is, the memory context that was current when SPI_connect was called, which is precisely the right context for a value returned from your - procedure. - - - - If SPI_palloc is called while the procedure is - not connected to SPI, then it acts the same as a normal - palloc. Before a procedure connects to the - SPI manager, the current memory context is the upper executor - context, so all allocations made by the procedure via - palloc or by SPI utility functions are made in - this context. + procedure. Several of the other utility procedures described in + this section also return objects created in the upper executor context. @@ -3379,25 +3268,14 @@ char * SPI_getnspname(Relation rel) context of the procedure, which is created by SPI_connect, is made the current context. All allocations made by palloc, - repalloc, or SPI utility functions (except for - SPI_copytuple, - SPI_returntuple, - SPI_modifytuple, - SPI_palloc, and - SPI_datumTransfer) are made in this context. When a + repalloc, or SPI utility functions (except as + described in this section) are made in this context. When a procedure disconnects from the SPI manager (via SPI_finish) the current context is restored to the upper executor context, and all allocations made in the procedure memory context are freed and cannot be used any more. - - All functions described in this section can be used by both - connected and unconnected procedures. In an unconnected procedure, - they act the same as the underlying ordinary server functions - (palloc, etc.). - - @@ -3426,6 +3304,11 @@ void * SPI_palloc(Size size) SPI_palloc allocates memory in the upper executor context. + + + This function can only be used while connected to SPI. + Otherwise, it throws an error. + @@ -3605,6 +3488,12 @@ HeapTuple SPI_copytuple(HeapTuple row) row from a trigger. In a function declared to return a composite type, use SPI_returntuple instead. + + + This function can only be used while connected to SPI. + Otherwise, it returns NULL and sets SPI_result to + SPI_ERROR_UNCONNECTED. + @@ -3626,8 +3515,8 @@ HeapTuple SPI_copytuple(HeapTuple row) Return Value - the copied row; NULL only if - tuple is NULL + the copied row, or NULL on error + (see SPI_result for an error indication) @@ -3663,6 +3552,12 @@ HeapTupleHeader SPI_returntuple(HeapTuple row, TupleDesc before returning. + + This function can only be used while connected to SPI. + Otherwise, it returns NULL and sets SPI_result to + SPI_ERROR_UNCONNECTED. + + Note that this should be used for functions that are declared to return composite types. It is not used for triggers; use @@ -3699,10 +3594,9 @@ HeapTupleHeader SPI_returntuple(HeapTuple row, TupleDesc Return Value - HeapTupleHeader pointing to copied row; - NULL only if - row or rowdesc is - NULL + HeapTupleHeader pointing to copied row, + or NULL on error + (see SPI_result for an error indication) @@ -3736,6 +3630,13 @@ HeapTuple SPI_modifytuple(Relation rel, HeapTuple SPI_modifytuple creates a new row by substituting new values for selected columns, copying the original row's columns at other positions. The input row is not modified. + The new row is returned in the upper executor context. + + + + This function can only be used while connected to SPI. + Otherwise, it returns NULL and sets SPI_result to + SPI_ERROR_UNCONNECTED. @@ -3821,8 +3722,8 @@ HeapTuple SPI_modifytuple(Relation rel, HeapTuple new row with modifications, allocated in the upper executor - context; NULL only if row - is NULL + context, or NULL on error + (see SPI_result for an error indication) @@ -3845,11 +3746,20 @@ HeapTuple SPI_modifytuple(Relation rel, HeapTuple if colnum contains an invalid column number (less - than or equal to 0 or greater than the number of column in + than or equal to 0 or greater than the number of columns in row) + + + SPI_ERROR_UNCONNECTED + + + if SPI is not active + + + diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index 8e650bc412..80fc4c4725 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -44,8 +44,7 @@ int SPI_result; static _SPI_connection *_SPI_stack = NULL; static _SPI_connection *_SPI_current = NULL; static int _SPI_stack_depth = 0; /* allocated size of _SPI_stack */ -static int _SPI_connected = -1; -static int _SPI_curid = -1; +static int _SPI_connected = -1; /* current stack index */ static Portal SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, ParamListInfo paramLI, bool read_only); @@ -86,13 +85,7 @@ SPI_connect(void) { int newdepth; - /* - * When procedure called by Executor _SPI_curid expected to be equal to - * _SPI_connected - */ - if (_SPI_curid != _SPI_connected) - return SPI_ERROR_CONNECT; - + /* Enlarge stack if necessary */ if (_SPI_stack == NULL) { if (_SPI_connected != -1 || _SPI_stack_depth != 0) @@ -117,9 +110,7 @@ SPI_connect(void) } } - /* - * We're entering procedure where _SPI_curid == _SPI_connected - 1 - */ + /* Enter new stack level */ _SPI_connected++; Assert(_SPI_connected >= 0 && _SPI_connected < _SPI_stack_depth); @@ -178,14 +169,9 @@ SPI_finish(void) SPI_lastoid = InvalidOid; SPI_tuptable = NULL; - /* - * After _SPI_begin_call _SPI_connected == _SPI_curid. Now we are closing - * connection to SPI and returning to upper Executor and so _SPI_connected - * must be equal to _SPI_curid. - */ + /* Exit stack level */ _SPI_connected--; - _SPI_curid--; - if (_SPI_connected == -1) + if (_SPI_connected < 0) _SPI_current = NULL; else _SPI_current = &(_SPI_stack[_SPI_connected]); @@ -212,7 +198,7 @@ AtEOXact_SPI(bool isCommit) _SPI_current = _SPI_stack = NULL; _SPI_stack_depth = 0; - _SPI_connected = _SPI_curid = -1; + _SPI_connected = -1; SPI_processed = 0; SPI_lastoid = InvalidOid; SPI_tuptable = NULL; @@ -258,8 +244,7 @@ AtEOSubXact_SPI(bool isCommit, SubTransactionId mySubid) * be already gone. */ _SPI_connected--; - _SPI_curid = _SPI_connected; - if (_SPI_connected == -1) + if (_SPI_connected < 0) _SPI_current = NULL; else _SPI_current = &(_SPI_stack[_SPI_connected]); @@ -313,53 +298,6 @@ AtEOSubXact_SPI(bool isCommit, SubTransactionId mySubid) } -/* Pushes SPI stack to allow recursive SPI calls */ -void -SPI_push(void) -{ - _SPI_curid++; -} - -/* Pops SPI stack to allow recursive SPI calls */ -void -SPI_pop(void) -{ - _SPI_curid--; -} - -/* Conditional push: push only if we're inside a SPI procedure */ -bool -SPI_push_conditional(void) -{ - bool pushed = (_SPI_curid != _SPI_connected); - - if (pushed) - { - _SPI_curid++; - /* We should now be in a state where SPI_connect would succeed */ - Assert(_SPI_curid == _SPI_connected); - } - return pushed; -} - -/* Conditional pop: pop only if SPI_push_conditional pushed */ -void -SPI_pop_conditional(bool pushed) -{ - /* We should be in a state where SPI_connect would succeed */ - Assert(_SPI_curid == _SPI_connected); - if (pushed) - _SPI_curid--; -} - -/* Restore state of SPI stack after aborting a subtransaction */ -void -SPI_restore_connection(void) -{ - Assert(_SPI_connected >= 0); - _SPI_curid = _SPI_connected - 1; -} - /* Parse, plan, and execute a query string */ int SPI_execute(const char *src, bool read_only, long tcount) @@ -691,7 +629,7 @@ SPI_freeplan(SPIPlanPtr plan) HeapTuple SPI_copytuple(HeapTuple tuple) { - MemoryContext oldcxt = NULL; + MemoryContext oldcxt; HeapTuple ctuple; if (tuple == NULL) @@ -700,17 +638,17 @@ SPI_copytuple(HeapTuple tuple) return NULL; } - if (_SPI_curid + 1 == _SPI_connected) /* connected */ + if (_SPI_current == NULL) { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); + SPI_result = SPI_ERROR_UNCONNECTED; + return NULL; } + oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); + ctuple = heap_copytuple(tuple); - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + MemoryContextSwitchTo(oldcxt); return ctuple; } @@ -718,7 +656,7 @@ SPI_copytuple(HeapTuple tuple) HeapTupleHeader SPI_returntuple(HeapTuple tuple, TupleDesc tupdesc) { - MemoryContext oldcxt = NULL; + MemoryContext oldcxt; HeapTupleHeader dtup; if (tuple == NULL || tupdesc == NULL) @@ -727,22 +665,22 @@ SPI_returntuple(HeapTuple tuple, TupleDesc tupdesc) return NULL; } + if (_SPI_current == NULL) + { + SPI_result = SPI_ERROR_UNCONNECTED; + return NULL; + } + /* For RECORD results, make sure a typmod has been assigned */ if (tupdesc->tdtypeid == RECORDOID && tupdesc->tdtypmod < 0) assign_record_type_typmod(tupdesc); - if (_SPI_curid + 1 == _SPI_connected) /* connected */ - { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); - } + oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); dtup = DatumGetHeapTupleHeader(heap_copy_tuple_as_datum(tuple, tupdesc)); - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + MemoryContextSwitchTo(oldcxt); return dtup; } @@ -751,7 +689,7 @@ HeapTuple SPI_modifytuple(Relation rel, HeapTuple tuple, int natts, int *attnum, Datum *Values, const char *Nulls) { - MemoryContext oldcxt = NULL; + MemoryContext oldcxt; HeapTuple mtuple; int numberOfAttributes; Datum *v; @@ -764,13 +702,16 @@ SPI_modifytuple(Relation rel, HeapTuple tuple, int natts, int *attnum, return NULL; } - if (_SPI_curid + 1 == _SPI_connected) /* connected */ + if (_SPI_current == NULL) { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); + SPI_result = SPI_ERROR_UNCONNECTED; + return NULL; } + + oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); + SPI_result = 0; + numberOfAttributes = rel->rd_att->natts; v = (Datum *) palloc(numberOfAttributes * sizeof(Datum)); n = (bool *) palloc(numberOfAttributes * sizeof(bool)); @@ -810,8 +751,7 @@ SPI_modifytuple(Relation rel, HeapTuple tuple, int natts, int *attnum, pfree(v); pfree(n); - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + MemoryContextSwitchTo(oldcxt); return mtuple; } @@ -980,22 +920,10 @@ SPI_getnspname(Relation rel) void * SPI_palloc(Size size) { - MemoryContext oldcxt = NULL; - void *pointer; - - if (_SPI_curid + 1 == _SPI_connected) /* connected */ - { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); - } - - pointer = palloc(size); - - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + if (_SPI_current == NULL) + elog(ERROR, "SPI_palloc called while not connected to SPI"); - return pointer; + return MemoryContextAlloc(_SPI_current->savedcxt, size); } void * @@ -1015,20 +943,17 @@ SPI_pfree(void *pointer) Datum SPI_datumTransfer(Datum value, bool typByVal, int typLen) { - MemoryContext oldcxt = NULL; + MemoryContext oldcxt; Datum result; - if (_SPI_curid + 1 == _SPI_connected) /* connected */ - { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); - } + if (_SPI_current == NULL) + elog(ERROR, "SPI_datumTransfer called while not connected to SPI"); + + oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); result = datumTransfer(value, typByVal, typLen); - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + MemoryContextSwitchTo(oldcxt); return result; } @@ -1050,17 +975,12 @@ SPI_freetuptable(SPITupleTable *tuptable) return; /* - * Since this function might be called during error recovery, it seems - * best not to insist that the caller be actively connected. We just - * search the topmost SPI context, connected or not. + * Search only the topmost SPI context for a matching tuple table. */ - if (_SPI_connected >= 0) + if (_SPI_current != NULL) { slist_mutable_iter siter; - if (_SPI_current != &(_SPI_stack[_SPI_connected])) - elog(ERROR, "SPI stack corrupted"); - /* find tuptable in active list, then remove it */ slist_foreach_modify(siter, &_SPI_current->tuptables) { @@ -1168,13 +1088,9 @@ SPI_cursor_open_with_args(const char *name, /* We needn't copy the plan; SPI_cursor_open_internal will do so */ - /* Adjust stack so that SPI_cursor_open_internal doesn't complain */ - _SPI_curid--; - result = SPI_cursor_open_internal(name, &plan, paramLI, read_only); /* And clean up */ - _SPI_curid++; _SPI_end_call(true); return result; @@ -1723,14 +1639,8 @@ spi_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo) MemoryContext oldcxt; MemoryContext tuptabcxt; - /* - * When called by Executor _SPI_curid expected to be equal to - * _SPI_connected - */ - if (_SPI_curid != _SPI_connected || _SPI_connected < 0) - elog(ERROR, "improper call to spi_dest_startup"); - if (_SPI_current != &(_SPI_stack[_SPI_curid])) - elog(ERROR, "SPI stack corrupted"); + if (_SPI_current == NULL) + elog(ERROR, "spi_dest_startup called while not connected to SPI"); if (_SPI_current->tuptable != NULL) elog(ERROR, "improper call to spi_dest_startup"); @@ -1775,14 +1685,8 @@ spi_printtup(TupleTableSlot *slot, DestReceiver *self) SPITupleTable *tuptable; MemoryContext oldcxt; - /* - * When called by Executor _SPI_curid expected to be equal to - * _SPI_connected - */ - if (_SPI_curid != _SPI_connected || _SPI_connected < 0) - elog(ERROR, "improper call to spi_printtup"); - if (_SPI_current != &(_SPI_stack[_SPI_curid])) - elog(ERROR, "SPI stack corrupted"); + if (_SPI_current == NULL) + elog(ERROR, "spi_printtup called while not connected to SPI"); tuptable = _SPI_current->tuptable; if (tuptable == NULL) @@ -2534,11 +2438,8 @@ _SPI_procmem(void) static int _SPI_begin_call(bool execmem) { - if (_SPI_curid + 1 != _SPI_connected) + if (_SPI_current == NULL) return SPI_ERROR_UNCONNECTED; - _SPI_curid++; - if (_SPI_current != &(_SPI_stack[_SPI_curid])) - elog(ERROR, "SPI stack corrupted"); if (execmem) /* switch to the Executor memory context */ _SPI_execmem(); @@ -2554,11 +2455,6 @@ _SPI_begin_call(bool execmem) static int _SPI_end_call(bool procmem) { - /* - * We're returning to procedure where _SPI_curid == _SPI_connected - 1 - */ - _SPI_curid--; - if (procmem) /* switch to the procedure memory context */ { _SPI_procmem(); diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c index b144920ec6..057c3bfd7c 100644 --- a/src/backend/utils/adt/xml.c +++ b/src/backend/utils/adt/xml.c @@ -2644,8 +2644,6 @@ schema_to_xml_internal(Oid nspid, const char *xmlschema, bool nulls, relid_list = schema_get_xml_visible_tables(nspid); - SPI_push(); - foreach(cell, relid_list) { Oid relid = lfirst_oid(cell); @@ -2658,7 +2656,6 @@ schema_to_xml_internal(Oid nspid, const char *xmlschema, bool nulls, appendStringInfoChar(result, '\n'); } - SPI_pop(); SPI_finish(); xmldata_root_element_end(result, xmlsn); @@ -2822,8 +2819,6 @@ database_to_xml_internal(const char *xmlschema, bool nulls, nspid_list = database_get_xml_visible_schemas(); - SPI_push(); - foreach(cell, nspid_list) { Oid nspid = lfirst_oid(cell); @@ -2836,7 +2831,6 @@ database_to_xml_internal(const char *xmlschema, bool nulls, appendStringInfoChar(result, '\n'); } - SPI_pop(); SPI_finish(); xmldata_root_element_end(result, xmlcn); diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index c96a86500a..884cdab702 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -53,7 +53,6 @@ #include "access/transam.h" #include "catalog/namespace.h" #include "executor/executor.h" -#include "executor/spi.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "optimizer/cost.h" @@ -878,7 +877,6 @@ BuildCachedPlan(CachedPlanSource *plansource, List *qlist, CachedPlan *plan; List *plist; bool snapshot_set; - bool spi_pushed; bool is_transient; MemoryContext plan_context; MemoryContext oldcxt = CurrentMemoryContext; @@ -926,22 +924,11 @@ BuildCachedPlan(CachedPlanSource *plansource, List *qlist, snapshot_set = true; } - /* - * The planner may try to call SPI-using functions, which causes a problem - * if we're already inside one. Rather than expect all SPI-using code to - * do SPI_push whenever a replan could happen, it seems best to take care - * of the case here. - */ - spi_pushed = SPI_push_conditional(); - /* * Generate the plan. */ plist = pg_plan_queries(qlist, plansource->cursor_options, boundParams); - /* Clean up SPI state */ - SPI_pop_conditional(spi_pushed); - /* Release snapshot if we got one */ if (snapshot_set) PopActiveSnapshot(); diff --git a/src/backend/utils/fmgr/fmgr.c b/src/backend/utils/fmgr/fmgr.c index 46a55ba7b9..3340b17d90 100644 --- a/src/backend/utils/fmgr/fmgr.c +++ b/src/backend/utils/fmgr/fmgr.c @@ -19,7 +19,6 @@ #include "catalog/pg_language.h" #include "catalog/pg_proc.h" #include "executor/functions.h" -#include "executor/spi.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" @@ -1878,25 +1877,16 @@ OidFunctionCall9Coll(Oid functionId, Oid collation, Datum arg1, Datum arg2, * the caller should assume the result is NULL, but we'll call the input * function anyway if it's not strict. So this is almost but not quite * the same as FunctionCall3. - * - * One important difference from the bare function call is that we will - * push any active SPI context, allowing SPI-using I/O functions to be - * called from other SPI functions without extra notation. This is a hack, - * but the alternative of expecting all SPI functions to do SPI_push/SPI_pop - * around I/O calls seems worse. */ Datum InputFunctionCall(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod) { FunctionCallInfoData fcinfo; Datum result; - bool pushed; if (str == NULL && flinfo->fn_strict) return (Datum) 0; /* just return null result */ - pushed = SPI_push_conditional(); - InitFunctionCallInfoData(fcinfo, flinfo, 3, InvalidOid, NULL, NULL); fcinfo.arg[0] = CStringGetDatum(str); @@ -1922,8 +1912,6 @@ InputFunctionCall(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod) fcinfo.flinfo->fn_oid); } - SPI_pop_conditional(pushed); - return result; } @@ -1932,22 +1920,12 @@ InputFunctionCall(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod) * * Do not call this on NULL datums. * - * This is almost just window dressing for FunctionCall1, but it includes - * SPI context pushing for the same reasons as InputFunctionCall. + * This is currently little more than window dressing for FunctionCall1. */ char * OutputFunctionCall(FmgrInfo *flinfo, Datum val) { - char *result; - bool pushed; - - pushed = SPI_push_conditional(); - - result = DatumGetCString(FunctionCall1(flinfo, val)); - - SPI_pop_conditional(pushed); - - return result; + return DatumGetCString(FunctionCall1(flinfo, val)); } /* @@ -1956,8 +1934,7 @@ OutputFunctionCall(FmgrInfo *flinfo, Datum val) * "buf" may be NULL to indicate we are reading a NULL. In this case * the caller should assume the result is NULL, but we'll call the receive * function anyway if it's not strict. So this is almost but not quite - * the same as FunctionCall3. Also, this includes SPI context pushing for - * the same reasons as InputFunctionCall. + * the same as FunctionCall3. */ Datum ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, @@ -1965,13 +1942,10 @@ ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, { FunctionCallInfoData fcinfo; Datum result; - bool pushed; if (buf == NULL && flinfo->fn_strict) return (Datum) 0; /* just return null result */ - pushed = SPI_push_conditional(); - InitFunctionCallInfoData(fcinfo, flinfo, 3, InvalidOid, NULL, NULL); fcinfo.arg[0] = PointerGetDatum(buf); @@ -1997,8 +1971,6 @@ ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, fcinfo.flinfo->fn_oid); } - SPI_pop_conditional(pushed); - return result; } @@ -2009,22 +1981,12 @@ ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, * * This is little more than window dressing for FunctionCall1, but it does * guarantee a non-toasted result, which strictly speaking the underlying - * function doesn't. Also, this includes SPI context pushing for the same - * reasons as InputFunctionCall. + * function doesn't. */ bytea * SendFunctionCall(FmgrInfo *flinfo, Datum val) { - bytea *result; - bool pushed; - - pushed = SPI_push_conditional(); - - result = DatumGetByteaP(FunctionCall1(flinfo, val)); - - SPI_pop_conditional(pushed); - - return result; + return DatumGetByteaP(FunctionCall1(flinfo, val)); } /* diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h index 1792fb1217..76ba394a2b 100644 --- a/src/include/executor/spi.h +++ b/src/include/executor/spi.h @@ -59,6 +59,13 @@ typedef struct _SPI_plan *SPIPlanPtr; #define SPI_OK_UPDATE_RETURNING 13 #define SPI_OK_REWRITTEN 14 +/* These used to be functions, now just no-ops for backwards compatibility */ +#define SPI_push() ((void) 0) +#define SPI_pop() ((void) 0) +#define SPI_push_conditional() false +#define SPI_pop_conditional(pushed) ((void) 0) +#define SPI_restore_connection() ((void) 0) + extern PGDLLIMPORT uint64 SPI_processed; extern PGDLLIMPORT Oid SPI_lastoid; extern PGDLLIMPORT SPITupleTable *SPI_tuptable; @@ -66,11 +73,6 @@ extern PGDLLIMPORT int SPI_result; extern int SPI_connect(void); extern int SPI_finish(void); -extern void SPI_push(void); -extern void SPI_pop(void); -extern bool SPI_push_conditional(void); -extern void SPI_pop_conditional(bool pushed); -extern void SPI_restore_connection(void); extern int SPI_execute(const char *src, bool read_only, long tcount); extern int SPI_execute_plan(SPIPlanPtr plan, Datum *Values, const char *Nulls, bool read_only, long tcount); diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 461986cda3..9a2d0527f8 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -3057,12 +3057,6 @@ plperl_spi_exec(char *query, int limit) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3078,13 +3072,6 @@ plperl_spi_exec(char *query, int limit) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3296,12 +3283,6 @@ plperl_spi_query(char *query) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3317,13 +3298,6 @@ plperl_spi_query(char *query) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3382,12 +3356,6 @@ plperl_spi_fetchrow(char *cursor) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3403,13 +3371,6 @@ plperl_spi_fetchrow(char *cursor) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3543,12 +3504,6 @@ plperl_spi_prepare(char *query, int argc, SV **argv) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3574,13 +3529,6 @@ plperl_spi_prepare(char *query, int argc, SV **argv) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3694,12 +3642,6 @@ plperl_spi_exec_prepared(char *query, HV *attr, int argc, SV **argv) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3715,13 +3657,6 @@ plperl_spi_exec_prepared(char *query, HV *attr, int argc, SV **argv) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3823,12 +3758,6 @@ plperl_spi_query_prepared(char *query, int argc, SV **argv) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3844,13 +3773,6 @@ plperl_spi_query_prepared(char *query, int argc, SV **argv) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index 91e1f8dd3f..77e7440002 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -1337,12 +1337,6 @@ exec_stmt_block(PLpgSQL_execstate *estate, PLpgSQL_stmt_block *block) * automatically cleaned up during subxact exit.) */ estate->eval_econtext = old_eval_econtext; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but - * just in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -1384,13 +1378,6 @@ exec_stmt_block(PLpgSQL_execstate *estate, PLpgSQL_stmt_block *block) /* Revert to outer eval_econtext */ estate->eval_econtext = old_eval_econtext; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it - * will have left us in a disconnected state. We need this hack - * to return to connected state. - */ - SPI_restore_connection(); - /* * Must clean up the econtext too. However, any tuple table made * in the subxact will have been thrown away by SPI during subxact @@ -5587,8 +5574,6 @@ exec_eval_simple_expr(PLpgSQL_execstate *estate, * Without this, stable functions within the expression would fail to see * updates made so far by our own function. */ - SPI_push(); - oldcontext = MemoryContextSwitchTo(get_eval_mcontext(estate)); if (!estate->readonly_func) { @@ -5636,8 +5621,6 @@ exec_eval_simple_expr(PLpgSQL_execstate *estate, MemoryContextSwitchTo(oldcontext); - SPI_pop(); - /* * Now we can release our refcount on the cached plan. */ @@ -6281,8 +6264,6 @@ exec_cast_value(PLpgSQL_execstate *estate, ExprContext *econtext = estate->eval_econtext; MemoryContext oldcontext; - SPI_push(); - oldcontext = MemoryContextSwitchTo(get_eval_mcontext(estate)); econtext->caseValue_datum = value; @@ -6296,8 +6277,6 @@ exec_cast_value(PLpgSQL_execstate *estate, cast_entry->cast_in_use = false; MemoryContextSwitchTo(oldcontext); - - SPI_pop(); } } diff --git a/src/pl/plpython/plpy_exec.c b/src/pl/plpython/plpy_exec.c index fa5b25a5fa..697a0e1cc0 100644 --- a/src/pl/plpython/plpy_exec.c +++ b/src/pl/plpython/plpy_exec.c @@ -1103,8 +1103,6 @@ PLy_abort_open_subtransactions(int save_subxact_level) RollbackAndReleaseCurrentSubTransaction(); - SPI_restore_connection(); - subtransactiondata = (PLySubtransactionData *) linitial(explicit_subtransactions); explicit_subtransactions = list_delete_first(explicit_subtransactions); diff --git a/src/pl/plpython/plpy_spi.c b/src/pl/plpython/plpy_spi.c index b082d017ea..07ab6a087e 100644 --- a/src/pl/plpython/plpy_spi.c +++ b/src/pl/plpython/plpy_spi.c @@ -516,12 +516,6 @@ PLy_spi_subtransaction_commit(MemoryContext oldcontext, ResourceOwner oldowner) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just in - * case it did, make sure we remain connected. - */ - SPI_restore_connection(); } void @@ -541,13 +535,6 @@ PLy_spi_subtransaction_abort(MemoryContext oldcontext, ResourceOwner oldowner) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return to - * connected state. - */ - SPI_restore_connection(); - /* Look up the correct exception */ entry = hash_search(PLy_spi_exceptions, &(edata->sqlerrcode), HASH_FIND, NULL); diff --git a/src/pl/plpython/plpy_subxactobject.c b/src/pl/plpython/plpy_subxactobject.c index 81fb3a3a4a..9f1caa87d9 100644 --- a/src/pl/plpython/plpy_subxactobject.c +++ b/src/pl/plpython/plpy_subxactobject.c @@ -7,7 +7,6 @@ #include "postgres.h" #include "access/xact.h" -#include "executor/spi.h" #include "utils/memutils.h" #include "plpython.h" @@ -213,12 +212,6 @@ PLy_subtransaction_exit(PyObject *self, PyObject *args) CurrentResourceOwner = subxactdata->oldowner; pfree(subxactdata); - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just in - * case it did, make sure we remain connected. - */ - SPI_restore_connection(); - Py_INCREF(Py_None); return Py_None; } diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 20809102ef..b0d9e419bb 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -2182,11 +2182,9 @@ pltcl_returnnext(ClientData cdata, Tcl_Interp *interp, { HeapTuple tuple; - SPI_push(); tuple = pltcl_build_tuple_result(interp, rowObjv, rowObjc, call_state); tuplestore_puttuple(call_state->tuple_store, tuple); - SPI_pop(); } } else @@ -2249,12 +2247,6 @@ pltcl_subtrans_commit(MemoryContext oldcontext, ResourceOwner oldowner) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just in - * case it did, make sure we remain connected. - */ - SPI_restore_connection(); } static void @@ -2273,13 +2265,6 @@ pltcl_subtrans_abort(Tcl_Interp *interp, MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return to - * connected state. - */ - SPI_restore_connection(); - /* Pass the error data to Tcl */ pltcl_construct_errorCode(interp, edata); UTF_BEGIN; @@ -3029,9 +3014,6 @@ pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc) * mess, there's no way to prevent the datatype input functions it calls * from leaking. Run it in a short-lived context, unless we're about to * exit the procedure anyway. - * - * Also, caller is responsible for doing SPI_push/SPI_pop if calling from - * inside SPI environment. **********************************************************************/ static HeapTuple pltcl_build_tuple_result(Tcl_Interp *interp, Tcl_Obj **kvObjv, int kvObjc, From 3887ba6dbb08f50c0ee6639a80e68ef697222457 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 9 Nov 2016 12:00:00 -0500 Subject: [PATCH 467/871] doc: Improve whitespace use in XSL --- doc/src/sgml/stylesheet-common.xsl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/stylesheet-common.xsl b/doc/src/sgml/stylesheet-common.xsl index e3841130eb..c23d38f128 100644 --- a/doc/src/sgml/stylesheet-common.xsl +++ b/doc/src/sgml/stylesheet-common.xsl @@ -77,7 +77,9 @@ - ?? + ? + + ? From 41124a91e61fc6d9681c1e8b15ba30494e84d643 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 9 Nov 2016 16:26:32 -0500 Subject: [PATCH 468/871] pgbench: Allow the transaction log file prefix to be changed. Masahiko Sawada, reviewed by Fabien Coelho and Beena Emerson, with some a bit of wordsmithing and cosmetic adjustment by me. --- doc/src/sgml/ref/pgbench.sgml | 26 +++++++++++++++++++------- src/bin/pgbench/pgbench.c | 20 ++++++++++++++++++-- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml index 285608d508..3a65729bf3 100644 --- a/doc/src/sgml/ref/pgbench.sgml +++ b/doc/src/sgml/ref/pgbench.sgml @@ -614,6 +614,16 @@ pgbench options dbname + + + + + Set the filename prefix for the transaction log file created by + + + + @@ -1121,15 +1131,17 @@ END; With the , pgbench writes the time taken by each transaction to a log file. The log file will be named - pgbench_log.nnn, where - nnn is the PID of the pgbench process. - If the