resolve merge conflicts

Author: funbringer

.travis.yml

@@ -10,19 +10,19 @@ services:
     - docker
 
 install:
-    - sed -e 's/${CHECK_CODE}/'${CHECK_CODE}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile.tmpl > Dockerfile
+    - echo "FROM ${DOCKER_IMAGE}" > Dockerfile
     - docker-compose build
 
 script:
     - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests
 
 env:
-    - PG_VERSION=10  CHECK_CODE=clang
-    - PG_VERSION=9.6 CHECK_CODE=clang
-    - PG_VERSION=9.5 CHECK_CODE=clang
-    - PG_VERSION=10  CHECK_CODE=cppcheck
-    - PG_VERSION=9.6 CHECK_CODE=cppcheck
-    - PG_VERSION=9.5 CHECK_CODE=cppcheck
-    - PG_VERSION=10  CHECK_CODE=false
-    - PG_VERSION=9.6 CHECK_CODE=false
-    - PG_VERSION=9.5 CHECK_CODE=false
+    - DOCKER_IMAGE=pathman/pg95_clang_check_code
+    - DOCKER_IMAGE=pathman/pg95_cppcheck
+    - DOCKER_IMAGE=pathman/pg95_pathman_tests
+    - DOCKER_IMAGE=pathman/pg96_clang_check_code
+    - DOCKER_IMAGE=pathman/pg96_cppcheck
+    - DOCKER_IMAGE=pathman/pg96_pathman_tests
+    - DOCKER_IMAGE=pathman/pg10_clang_check_code
+    - DOCKER_IMAGE=pathman/pg10_cppcheck
+    - DOCKER_IMAGE=pathman/pg10_pathman_tests

Dockerfile.tmpl

@@ -14,7 +14,7 @@ RUN if [ "${CHECK_CODE}" = "cppcheck" ] ; then \
 RUN if [ "${CHECK_CODE}" = "false" ] ; then \
 	echo 'http://dl-3.alpinelinux.org/alpine/edge/main' > /etc/apk/repositories; \
 	apk --no-cache add curl python3 gcc make musl-dev cmocka-dev;\
-	pip3 install testgres; \
+	pip3 install virtualenv;\
 	fi
 
 RUN mkdir -p /pg/data && \
@@ -23,8 +23,8 @@ RUN mkdir -p /pg/data && \
 	chmod a+rwx /usr/local/lib/postgresql && \
 	chmod a+rwx /usr/local/share/postgresql/extension
 
-ADD . /pg/pg_pathman
-WORKDIR /pg/pg_pathman
-RUN chmod -R go+rwX /pg/pg_pathman
-USER postgres
-ENTRYPOINT PGDATA=${PGDATA} CHECK_CODE=${CHECK_CODE} bash run_tests.sh
+ONBUILD ADD . /pg/pg_pathman
+ONBUILD WORKDIR /pg/pg_pathman
+ONBUILD RUN chmod -R go+rwX /pg/pg_pathman
+ONBUILD USER postgres
+ONBUILD ENTRYPOINT PGDATA=${PGDATA} CHECK_CODE=${CHECK_CODE} bash run_tests.sh

README.md

@@ -251,6 +251,25 @@ drop_partitions(parent      REGCLASS,
 ```
 Drop partitions of the `parent` table (both foreign and local relations). If `delete_data` is `false`, the data is copied to the parent table first. Default is `false`.
 
+To remove partitioned table along with all partitions fully, use conventional
+`DROP TABLE relation CASCADE`. However, care should be taken in somewhat rare
+case when you are running logical replication and `DROP` was executed by
+replication apply worker, e.g. via trigger on replicated table. `pg_pathman`
+uses `pathman_ddl_trigger` event trigger to remove the record about dropped
+table from `pathman_config`, and this trigger by default won't fire on replica,
+leading to inconsistent state when `pg_pathman` thinks that the table still
+exists, but in fact it doesn't. If this is the case, configure this trigger to
+fire on replica too:
+
+```plpgsql
+ALTER EVENT TRIGGER pathman_ddl_trigger ENABLE ALWAYS;
+```
+
+Physical replication doesn't have this problem since DDL as well as
+`pathman_config` table is replicated too; master and slave PostgreSQL instances
+are basically identical, and it is only harmful to keep this trigger in `ALWAYS`
+mode.
+
 
 ### Additional parameters
 

expected/pathman_basic.out

@@ -1346,6 +1346,34 @@ NOTICE:  999 rows copied from test.num_range_rel_6
 DROP TABLE test.num_range_rel CASCADE;
 DROP TABLE test.range_rel CASCADE;
 NOTICE:  drop cascades to 10 other objects
+/* Test attributes copying */
+CREATE UNLOGGED TABLE test.range_rel (
+	id	SERIAL PRIMARY KEY,
+	dt	DATE NOT NULL)
+WITH (fillfactor = 70);
+INSERT INTO test.range_rel (dt)
+	SELECT g FROM generate_series('2015-01-01', '2015-02-15', '1 month'::interval) AS g;
+SELECT pathman.create_range_partitions('test.range_rel', 'dt',
+	'2015-01-01'::date, '1 month'::interval);
+ create_range_partitions 
+-------------------------
+                       2
+(1 row)
+
+SELECT reloptions, relpersistence FROM pg_class WHERE oid='test.range_rel'::REGCLASS;
+   reloptions    | relpersistence 
+-----------------+----------------
+ {fillfactor=70} | u
+(1 row)
+
+SELECT reloptions, relpersistence FROM pg_class WHERE oid='test.range_rel_1'::REGCLASS;
+   reloptions    | relpersistence 
+-----------------+----------------
+ {fillfactor=70} | u
+(1 row)
+
+DROP TABLE test.range_rel CASCADE;
+NOTICE:  drop cascades to 3 other objects
 /* Test automatic partition creation */
 CREATE TABLE test.range_rel (
 	id	SERIAL PRIMARY KEY,
@@ -1664,7 +1692,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM test.range_rel WHERE dt > '2010-12-15';
 CREATE TABLE test.tmp (id INTEGER NOT NULL, value INTEGER NOT NULL);
 INSERT INTO test.tmp VALUES (1, 1), (2, 2);
 /* Test UPDATE and DELETE */
-EXPLAIN (COSTS OFF) UPDATE test.range_rel SET value = 111 WHERE dt = '2010-06-15';
+EXPLAIN (COSTS OFF) UPDATE test.range_rel SET value = 111 WHERE dt = '2010-06-15';	/* have partitions for this 'dt' */
                                    QUERY PLAN                                   
 --------------------------------------------------------------------------------
  Update on range_rel_6
@@ -1679,7 +1707,7 @@ SELECT * FROM test.range_rel WHERE dt = '2010-06-15';
  166 | Tue Jun 15 00:00:00 2010 |   111
 (1 row)
 
-EXPLAIN (COSTS OFF) DELETE FROM test.range_rel WHERE dt = '2010-06-15';
+EXPLAIN (COSTS OFF) DELETE FROM test.range_rel WHERE dt = '2010-06-15';	/* have partitions for this 'dt' */
                                    QUERY PLAN                                   
 --------------------------------------------------------------------------------
  Delete on range_rel_6
@@ -1693,6 +1721,34 @@ SELECT * FROM test.range_rel WHERE dt = '2010-06-15';
 ----+----+-------
 (0 rows)
 
+EXPLAIN (COSTS OFF) UPDATE test.range_rel SET value = 222 WHERE dt = '1990-01-01';	/* no partitions for this 'dt' */
+                                   QUERY PLAN                                   
+--------------------------------------------------------------------------------
+ Update on range_rel
+   ->  Seq Scan on range_rel
+         Filter: (dt = 'Mon Jan 01 00:00:00 1990'::timestamp without time zone)
+(3 rows)
+
+UPDATE test.range_rel SET value = 111 WHERE dt = '1990-01-01';
+SELECT * FROM test.range_rel WHERE dt = '1990-01-01';
+ id | dt | value 
+----+----+-------
+(0 rows)
+
+EXPLAIN (COSTS OFF) DELETE FROM test.range_rel WHERE dt < '1990-01-01';	/* no partitions for this 'dt' */
+                                   QUERY PLAN                                   
+--------------------------------------------------------------------------------
+ Delete on range_rel
+   ->  Seq Scan on range_rel
+         Filter: (dt < 'Mon Jan 01 00:00:00 1990'::timestamp without time zone)
+(3 rows)
+
+DELETE FROM test.range_rel WHERE dt < '1990-01-01';
+SELECT * FROM test.range_rel WHERE dt < '1990-01-01';
+ id | dt | value 
+----+----+-------
+(0 rows)
+
 EXPLAIN (COSTS OFF) UPDATE test.range_rel r SET value = t.value FROM test.tmp t WHERE r.dt = '2010-01-01' AND r.id = t.id;
                                          QUERY PLAN                                         
 --------------------------------------------------------------------------------------------

expected/pathman_bgw.out

@@ -105,5 +105,43 @@ SELECT * FROM pathman_partition_list ORDER BY partition; /* should contain 3 par
 
 DROP TABLE test_bgw.test_4 CASCADE;
 NOTICE:  drop cascades to 4 other objects
+/* test error handling in BGW */
+CREATE TABLE test_bgw.test_5(val INT4 NOT NULL);
+SELECT create_range_partitions('test_bgw.test_5', 'val', 1, 10, 2);
+ create_range_partitions 
+-------------------------
+                       2
+(1 row)
+
+CREATE OR REPLACE FUNCTION test_bgw.abort_xact(args JSONB)
+RETURNS VOID AS $$
+BEGIN
+	RAISE EXCEPTION 'aborting xact!';
+END
+$$ language plpgsql;
+SELECT set_spawn_using_bgw('test_bgw.test_5', true);
+ set_spawn_using_bgw 
+---------------------
+ 
+(1 row)
+
+SELECT set_init_callback('test_bgw.test_5', 'test_bgw.abort_xact(jsonb)');
+ set_init_callback 
+-------------------
+ 
+(1 row)
+
+INSERT INTO test_bgw.test_5 VALUES (-100);
+ERROR:  Attempt to spawn new partitions of relation "test_5" failed
+SELECT * FROM pathman_partition_list ORDER BY partition; /* should contain 3 partitions */
+     parent      |     partition     | parttype | expr | range_min | range_max 
+-----------------+-------------------+----------+------+-----------+-----------
+ test_bgw.test_5 | test_bgw.test_5_1 |        2 | val  | 1         | 11
+ test_bgw.test_5 | test_bgw.test_5_2 |        2 | val  | 11        | 21
+(2 rows)
+
+DROP FUNCTION test_bgw.abort_xact(args JSONB);
+DROP TABLE test_bgw.test_5 CASCADE;
+NOTICE:  drop cascades to 3 other objects
 DROP SCHEMA test_bgw CASCADE;
 DROP EXTENSION pg_pathman;

expected/pathman_callbacks.out

@@ -1,14 +1,14 @@
 \set VERBOSITY terse
 CREATE EXTENSION pg_pathman;
 CREATE SCHEMA callbacks;
-/* Check callbacks */
+/* callback #1 */
 CREATE OR REPLACE FUNCTION callbacks.abc_on_part_created_callback(args JSONB)
 RETURNS VOID AS $$
 BEGIN
 	RAISE WARNING 'callback arg: %', args::TEXT;
 END
 $$ language plpgsql;
-/* callback is in public namespace, must be schema-qualified */
+/* callback #2 */
 CREATE OR REPLACE FUNCTION public.dummy_cb(args JSONB)
 RETURNS VOID AS $$
 BEGIN
@@ -184,15 +184,11 @@ WARNING:  callback arg: {"parent": "abc", "parttype": "2", "partition": "abc_2",
 
 INSERT INTO callbacks.abc VALUES (201, 0); /* +1 new partition */
 WARNING:  callback arg: {"parent": "abc", "parttype": "2", "partition": "abc_3", "range_max": "301", "range_min": "201", "parent_schema": "callbacks", "partition_schema": "callbacks"}
+BEGIN;
 DROP FUNCTION callbacks.abc_on_part_created_callback(jsonb);
 INSERT INTO callbacks.abc VALUES (301, 0); /* +0 new partitions (ERROR) */
 ERROR:  callback function "callbacks.abc_on_part_created_callback(jsonb)" does not exist
-CREATE OR REPLACE FUNCTION callbacks.abc_on_part_created_callback(args JSONB)
-RETURNS VOID AS $$
-BEGIN
-	RAISE WARNING 'callback arg: %', args::TEXT;
-END
-$$ language plpgsql;
+ROLLBACK;
 INSERT INTO callbacks.abc VALUES (301, 0); /* +1 new partition */
 WARNING:  callback arg: {"parent": "abc", "parttype": "2", "partition": "abc_5", "range_max": "401", "range_min": "301", "parent_schema": "callbacks", "partition_schema": "callbacks"}
 DROP TABLE callbacks.abc CASCADE;
@@ -211,22 +207,22 @@ CREATE OR REPLACE FUNCTION callbacks.rotation_callback(params jsonb)
 RETURNS VOID AS
 $$
 DECLARE
-    relation	regclass;
+	relation	regclass;
 	parent_rel	regclass;
 BEGIN
 	parent_rel := concat(params->>'partition_schema', '.', params->>'parent')::regclass;
 
-    -- drop "old" partitions
-    FOR relation IN (SELECT partition FROM
+	-- drop "old" partitions
+	FOR relation IN (SELECT partition FROM
 						(SELECT partition, range_min::INT4 FROM pathman_partition_list
 						 WHERE parent = parent_rel
 						 ORDER BY range_min::INT4 DESC
 						 OFFSET 4) t  -- remain 4 last partitions
 					 ORDER BY range_min)
-    LOOP
-        RAISE NOTICE 'dropping partition %', relation;
-        PERFORM drop_range_partition(relation);
-    END LOOP;
+	LOOP
+		RAISE NOTICE 'dropping partition %', relation;
+		PERFORM drop_range_partition(relation);
+	END LOOP;
 END
 $$ LANGUAGE plpgsql;
 SELECT * FROM pathman_partition_list

make_images.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+import subprocess
+import getpass
+
+DOCKER_ID = 'pathman'
+pg_versions = ['9.5','9.6','10']
+
+image_types = {
+	'clang_check_code': {
+		'CHECK_CODE': 'clang',
+	},
+	'cppcheck': {
+		'CHECK_CODE': 'cppcheck',
+	},
+	'pathman_tests': {
+		'CHECK_CODE': 'false',
+	}
+}
+
+user = input("Enter username for `docker login`: ")
+password = getpass.getpass()
+subprocess.check_output([
+	'docker',
+	'login',
+	'-u', user,
+	'-p', password])
+
+travis_conf_line = '- DOCKER_IMAGE=%s'
+travis_conf = []
+print("")
+
+for pg_version in pg_versions:
+	pgname = 'pg%s' % pg_version.replace('.', '')
+	for key, variables in image_types.items():
+		image_name = '%s/%s_%s' % (DOCKER_ID, pgname, key)
+		with open('Dockerfile', 'w') as out:
+			with open('Dockerfile.tmpl', 'r') as f:
+				for line in f:
+					line = line.replace('${PG_VERSION}', pg_version)
+					for key, value in variables.items():
+						varname = '${%s}' % key
+						line = line.replace(varname, value)
+
+					out.write(line)
+
+		args = [
+			'docker',
+			'build',
+			'-t', image_name,
+			'.'
+		]
+		subprocess.check_output(args, stderr=subprocess.STDOUT)
+		print("build ok:", image_name)
+		subprocess.check_output(['docker', 'push', image_name],
+				stderr=subprocess.STDOUT)
+		print("upload ok:", image_name)
+		travis_conf.append(travis_conf_line % image_name)
+
+print("\ntravis configuration")
+print('\n'.join(travis_conf))

run_tests.sh

@@ -35,6 +35,13 @@ elif [ "$CHECK_CODE" = "cppcheck" ]; then
     exit $status
 fi
 
+# we need testgres for pathman tests
+virtualenv env
+export VIRTUAL_ENV_DISABLE_PROMPT=1
+source env/bin/activate
+pip3 install testgres
+pip3 freeze | grep testgres
+
 # don't forget to "make clean"
 make USE_PGXS=1 clean