Fix/763: Tidy up database on migration

.github/actions/run-tests/reset-from-container.sh

@@ -1,4 +1,4 @@
-#! /bin/sh -e
+#! /bin/bash -e
 
 BACKUP="$1"
 
@@ -13,8 +13,30 @@ case "$INPUT_DB" in
 		mysql -u root -p"$MYSQL_ROOT_PASSWORD" -h mysql < "/dumps/$BACKUP/sql/dump.sql"
 		;;
 	pgsql)
+		echo 'Dropping old data'
 		PGPASSWORD="$POSTGRES_PASSWORD" \
-		psql -d "$POSTGRES_DB" -h postgres -U "$POSTGRES_USER" < "/dumps/$BACKUP/sql/dump.sql"
+		psql -d "$POSTGRES_DB" -h postgres -U "$POSTGRES_USER" -v 'ON_ERROR_STOP=1' <<- EOF || exit 1
+			DROP SCHEMA public CASCADE;
+			CREATE SCHEMA public;
+			GRANT ALL ON SCHEMA public TO $POSTGRES_USER;
+			GRANT ALL ON SCHEMA public TO public;
+			EOF
+
+# 		PGPASSWORD="$POSTGRES_PASSWORD" \
+# 		psql -d "$POSTGRES_DB" -h postgres -U "$POSTGRES_USER" <<- EOF
+# 			\l
+# 			\d
+# 			SELECT * FROM oc_preferences WHERE appid='cookbook';
+# 			EOF
+
+		echo 'Inserting dump data'
+		PGPASSWORD="$POSTGRES_PASSWORD" \
+		psql -d "$POSTGRES_DB" -h postgres -U "$POSTGRES_USER" -f "/dumps/$BACKUP/sql/dump.sql" -v 'ON_ERROR_STOP=1' || exit 1
+
+# 		PGPASSWORD="$POSTGRES_PASSWORD" \
+# 		psql -d "$POSTGRES_DB" -h postgres -U "$POSTGRES_USER" <<- EOF
+# 			SELECT * FROM oc_preferences WHERE appid='cookbook';
+# 			EOF
 		;;
 	sqlite)
 		echo "Doing nothing as it was already restored during data restoration."

.github/actions/run-tests/run-locally.sh

@@ -274,10 +274,12 @@ drop_environment(){
 			fi
 			;;
 		pgsql)
-			local tables=$(echo "SELECT tablename FROM pg_tables WHERE schemaname = 'public';" | call_postgres | head -n -1 )
-			if [ -n "$tables" ]; then
-				echo "$tables" | sed 's@.*@DROP TABLE \0;@' | call_postgres
-			fi
+			call_postgres <<- EOF
+				DROP SCHEMA public CASCADE;
+				CREATE SCHEMA public;
+				GRANT ALL ON SCHEMA public TO $POSTGRES_USER;
+				GRANT ALL ON SCHEMA public TO public;
+				EOF
 			;;
 	esac
 
@@ -337,6 +339,12 @@ restore_env_dump() {
 			;;
 		pgsql)
 			echo "Restoring Postgres database from dump"
+			call_postgres <<- EOF
+				DROP SCHEMA public CASCADE;
+				CREATE SCHEMA public;
+				GRANT ALL ON SCHEMA public TO $POSTGRES_USER;
+				GRANT ALL ON SCHEMA public TO public;
+				EOF
 			cat "volumes/dumps/$ENV_DUMP_PATH/sql/dump.sql" | call_postgres
 			;;
 		sqlite)
@@ -424,7 +432,7 @@ function call_mysql() {
 
 function call_postgres() {
 	#docker-compose exec -T postgres psql -t nc_test tester
-	docker-compose exec -T postgres psql -U "$POSTGRES_USER" "$POSTGRES_DB"
+	docker-compose exec -T postgres psql -U "$POSTGRES_USER" "$POSTGRES_DB" -v 'ON_ERROR_STOP=1' --quiet "$@"
 }
 
 ##### Parameters as extracted from the CLI

.github/actions/run-tests/run-occ.sh

@@ -0,0 +1,9 @@
+#! /bin/sh -e
+
+#set -x
+
+cd /nextcloud
+php occ "$@"
+
+exit $?
+

CHANGELOG.md

@@ -7,6 +7,8 @@
 ### Fixed
 - Fixed changes from #774 and minor extensions
   [#775](https://github.com/nextcloud/cookbook/pull/775) @christianlupus
+- Clean tables from old, redundant, and non-unique data to allow migrations (see #762 #763)
+  [#776](https://github.com/nextcloud/cookbook/pull/776) @christianlupus
 
 
 ## 0.9.1 - 2021-07-05

lib/Migration/Version000000Date20210701093123.php

@@ -4,16 +4,84 @@
 
 namespace OCA\Cookbook\Migration;
 
-use Closure;
+use OCP\IDBConnection;
 use OCP\DB\ISchemaWrapper;
 use OCP\Migration\IOutput;
 use OCP\Migration\SimpleMigrationStep;
+use Closure;
+use OCP\DB\QueryBuilder\IQueryBuilder;
 
 /**
  * Auto-generated migration step: Please modify to your needs!
  */
 class Version000000Date20210701093123 extends SimpleMigrationStep {
 
+	/**
+	 * @var IDBConnection
+	 */
+	private $db;
+
+	public function __construct(IDBConnection $db) {
+		$this->db = $db;
+	}
+
+	public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options) {
+		$this->db->beginTransaction();
+		try {
+			$qb = $this->db->getQueryBuilder();
+
+			// Fetch all rows that are non-unique
+			$qb->selectAlias('n.user_id', 'user')
+				->selectAlias('n.recipe_id', 'recipe')
+				->from('cookbook_names', 'n')
+				->groupBy('n.user_id', 'n.recipe_id')
+				->having('COUNT(*) > 1');
+			//echo $qb->getSQL() . "\n";
+
+			$cursor = $qb->execute();
+			$result = $cursor->fetchAll();
+
+			if (sizeof($result) > 0) {
+				// We have to fix the database
+
+				// Drop all redundant rows
+				$qb = $this->db->getQueryBuilder();
+				$qb->delete('cookbook_names')
+					->where(
+						'user_id = :user',
+						'recipe_id = :recipe'
+						);
+
+				$qb2 = $this->db->getQueryBuilder();
+				$qb2->update('preferences')
+					->set('configvalue', $qb->expr()->literal('1', IQueryBuilder::PARAM_STR))
+					->where(
+						'userid = :user',
+						'appid = :app',
+						'configkey = :property'
+						);
+				$qb2->setParameter('app', 'cookbook');
+				$qb2->setParameter('property', 'last_index_update');
+
+				foreach ($result as $r) {
+					$qb->setParameter('user', $r['user']);
+					$qb->setParameter('recipe', $r['recipe']);
+					$qb->execute();
+
+					$qb2->setParameter('user', $r['user']);
+					$qb2->execute();
+				}
+			}
+
+			// Finish the transaction
+			$this->db->commit();
+		} catch (\Exception $e) {
+			// Abort the transaction
+			$this->db->rollBack();
+			throw $e;
+		}
+	}
+
 	/**
 	 * @param IOutput $output
 	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`

phpunit.integration.xml

@@ -1,17 +1,19 @@
-<phpunit bootstrap="tests/bootstrap.php" colors="true">
+<phpunit 
+	bootstrap="tests/bootstrap.php"
+	colors="true"
+	backupGlobals="false"
+	backupStaticAttributes="false"
+	cacheResult="true"
+	cacheResultFile="/tmp/phpunit.cache"
+	>
     <testsuites>
         <testsuite name="integration">
             <directory>./tests/Integration</directory>
         </testsuite>
     </testsuites>
 	<filter>
 		<whitelist processUncoveredFilesFromWhitelist="true" addUncoveredFilesFromWhitelist="true">
-			<directory suffix=".php">lib/</directory>
-			<!--<directory suffix=".php">lib/Controller</directory>
-			<directory suffix=".php">lib/Db</directory>
-			<directory suffix=".php">lib/Exception</directory>
-			<directory suffix=".php">lib/Migration</directory>
-			<directory suffix=".php">lib/Service</directory>-->
+			<directory suffix=".php">lib</directory>
 		</whitelist>
 	</filter>
 </phpunit>

phpunit.xml

@@ -1,4 +1,11 @@
-<phpunit bootstrap="tests/bootstrap.php" colors="true">
+<phpunit 
+	bootstrap="tests/bootstrap.php"
+	colors="true"
+	backupGlobals="false"
+	backupStaticAttributes="false"
+	cacheResult="true"
+	cacheResultFile="/tmp/phpunit.cache"
+	>
     <testsuites>
         <testsuite name="unit">
             <directory>./tests/Unit</directory>
@@ -7,11 +14,6 @@
 	<filter>
 		<whitelist processUncoveredFilesFromWhitelist="true" addUncoveredFilesFromWhitelist="true">
 			<directory suffix=".php">lib</directory>
-			<!--<directory suffix=".php">lib/Controller</directory>
-			<directory suffix=".php">lib/Db</directory>
-			<directory suffix=".php">lib/Exception</directory>
-			<directory suffix=".php">lib/Migration</directory>
-			<directory suffix=".php">lib/Service</directory>-->
 		</whitelist>
 	</filter>
 </phpunit>