Description: Fix SQLite DB schema upgrade
Author: Marcin Deranek <marcin.deranek@slonko.net>
Origin: other, https://sourcesup.cru.fr/tracker/download.php/23/167/7571/1004/sympa-6.1.sqlite.patch
Bug: https://sourcesup.cru.fr/tracker/?group_id=23&atid=167&func=detail&aid=7571
Bug-Debian: http://bugs.debian.org/642464
Last-Update: 2011-11-19
--- a/src/lib/Upgrade.pm
+++ b/src/lib/Upgrade.pm
@@ -902,7 +902,7 @@
 							 'bounce_score_subscriber' => 'integer',
 							 'bounce_address_subscriber' => 'text',
 							 'custom_attribute_subscriber' => 'text',
-							 'suspend_subscriber' => "boolean",
+							 'suspend_subscriber' => 'numeric',
 							 'suspend_start_date_subscriber' => 'integer',
 							 'suspend_end_date_subscriber' => 'integer'},
 				  'admin_table' => {'list_admin' => 'text',
@@ -963,8 +963,8 @@
 							 'returnpath_bulkmailer' => 'text',
 							 'robot_bulkmailer' => 'text',
 							 'listname_bulkmailer' => 'text',
-							 'verp_bulkmailer' => 'integer',
-							 'merge_bulkmailer' => 'integer',
+							 'verp_bulkmailer' => 'numeric',
+							 'merge_bulkmailer' => 'numeric',
 							 'priority_message_bulkmailer' => 'integer',
 							 'priority_packet_bulkmailer' => 'integer',
 							 'reception_date_bulkmailer' => 'integer',
@@ -973,26 +973,26 @@
 				  'bulkspool_table' => {'messagekey_bulkspool' => 'text',
 							'messageid_bulkspool' => 'text',
 							'message_bulkspool' => 'text',
-							'lock_bulkspool' => 'integer',
-							'dkim_privatekey_bulkspool' => 'varchar(1000)',
-							'dkim_selector_bulkspool' => 'varchar(50)',
-							'dkim_d_bulkspool' => 'varchar(50)',
-							'dkim_i_bulkspool' => 'varchar(100)',
-							'dkim_header_list_bulkspool' => 'varchar(500)'},
+							'lock_bulkspool' => 'numeric',
+							'dkim_privatekey_bulkspool' => 'text',
+							'dkim_selector_bulkspool' => 'text',
+							'dkim_d_bulkspool' => 'text',
+							'dkim_i_bulkspool' => 'text',
+							'dkim_header_list_bulkspool' => 'text'},
 				  'conf_table' => {'robot_conf' => 'text',
 						   'label_conf' => 'text',
 						   'value_conf' => 'text'},
-                  'list_table' => {'name_list'=>'varchar(100)',
-                                    'path_list'=>'varchar(100)',
-                                    'robot_list'=>'varchar(100)',
-                                    'status_list'=>"enum('open','closed','pending','error_config','family_closed')",
-                                    'creation_email_list'=>'varchar(100)',
-                                    'creation_epoch_list'=>'datetime',
-                                    'subject_list'=>'varchar(100)',
-                                    'web_archive_list'=>'tinyint(1)',
-                                    'topics_list'=>'varchar(100)',
-                                    'editors_list'=>'varchar(100)',
-                                    'owners_list'=>'varchar(100)'}
+                  'list_table' => {'name_list' => 'text',
+                                    'path_list' => 'text',
+                                    'robot_list' => 'text',
+                                    'status_list' => 'text',
+                                    'creation_email_list' => 'text',
+                                    'creation_epoch_list' => 'numeric',
+                                    'subject_list' => 'text',
+                                    'web_archive_list' => 'integer',
+                                    'topics_list' => 'text',
+                                    'editors_list' => 'text',
+                                    'owners_list' => 'text'}
 });
 
     my %not_null = ('email_user' => 1,
@@ -1005,7 +1005,6 @@
 		    'user_admin' => 1,
 		    'role_admin' => 1,
 		    'date_admin' => 1,
-		    'list_exclusion' => 1,
 		    'user_exclusion' => 1,
 		    'robot_exclusion' => 1,
 		    'netid_netidmap' => 1,
@@ -1022,16 +1021,8 @@
 		    'messagekey_bulkmailer' => 1,
 		    'packetid_bulkmailer' => 1,
 		    'messagekey_bulkspool' => 1,
-             'name_list'=>1,
-             'path_list'=>1,
-             'robot_list'=>1,
-             'status_list'=>1,
-             'creation_email_list'=>1,
-             'subject_list'=>1,
-             'web_archive_list'=>1,
-             'topics_list'=>1,
-             'owners_list'=>1,
-             'editors_list'=>1
+		    'name_list'=>1,
+		    'robot_list'=>1,
 		    );
     
     my %primary = ('user_table' => ['email_user'],
@@ -1158,7 +1149,27 @@
 	    $t =~ s/^"main"\.//; # needed for SQLite 3
 	    $t =~ s/^.*\"([^"]+)\"$/$1/;
  	}
-	
+
+    ## Check required tables
+    foreach my $t1 (keys %{$db_struct{'SQLite'}}) {
+        my $found;
+        foreach my $t2 (@tables) {
+        $found = 1 if ($t1 eq $t2);
+        }
+        unless ($found) {
+        unless ($dbh->do("CREATE TABLE $t1 (temporary integer)")) {
+            &do_log('err', 'Could not create table %s in database %s : %s', $t1, $Conf::Conf{'db_name'}, $dbh->errstr);
+            next;
+        }
+        
+        push @report, sprintf('Table %s created in database %s', $t1, $Conf::Conf{'db_name'});
+        &do_log('notice', 'Table %s created in database %s', $t1, $Conf::Conf{'db_name'});
+        push @tables, $t1;
+        $real_struct{$t1} = {};
+        }
+    }
+
+    ## Get fields
 	foreach my $t (@tables) {
 	    next unless (defined $db_struct{$Conf::Conf{'db_type'}}{$t});
 	    
@@ -1265,7 +1276,10 @@
 		    my $options;
 		    ## To prevent "Cannot add a NOT NULL column with default value NULL" errors
 		    if ($not_null{$f}) {
-			$options .= 'NOT NULL';
+				$options .= 'NOT NULL';
+				if ($Conf::Conf{'db_type'} eq 'SQLite') {
+					$options .= ' DEFAULT ""';
+				}
 		    }
 		    
 		    unless ($dbh->do("ALTER TABLE $t ADD $f $db_struct{$Conf::Conf{'db_type'}}{$t}{$f} $options")) {
@@ -1278,6 +1292,7 @@
 		    &do_log('info', 'Field %s added to table %s', $f, $t);
 		    $added_fields{$f} = 1;
 		    
+            if ($Conf::Conf{'db_type'} eq 'mysql') {
 		    ## Remove temporary DB field
 		    if ($real_struct{$t}{'temporary'}) {
 			unless ($dbh->do("ALTER TABLE $t DROP temporary")) {
@@ -1285,43 +1300,93 @@
 			}
 			delete $real_struct{$t}{'temporary'};
 		    }
+            }
 		    
 		    next;
 		}
 		
 		## Change DB types if different and if update_db_types enabled
-		if ($Conf::Conf{'update_db_field_types'} eq 'auto' && $Conf::Conf{'db_type'} ne 'SQLite') {
-		    unless (&check_db_field_type(effective_format => $real_struct{$t}{$f},
-						 required_format => $db_struct{$Conf::Conf{'db_type'}}{$t}{$f})) {
-			push @report, sprintf('Field \'%s\'  (table \'%s\' ; database \'%s\') does NOT have awaited type (%s). Attempting to change it...', 
-					      $f, $t, $Conf::Conf{'db_name'}, $db_struct{$Conf::Conf{'db_type'}}{$t}{$f});
-			&do_log('notice', 'Field \'%s\'  (table \'%s\' ; database \'%s\') does NOT have awaited type (%s). Attempting to change it...', 
-				$f, $t, $Conf::Conf{'db_name'}, $db_struct{$Conf::Conf{'db_type'}}{$t}{$f});
-			
-			my $options;
-			if ($not_null{$f}) {
-			    $options .= 'NOT NULL';
-			}
-			
-			push @report, sprintf("ALTER TABLE $t CHANGE $f $f $db_struct{$Conf::Conf{'db_type'}}{$t}{$f} $options");
-			&do_log('notice', "ALTER TABLE $t CHANGE $f $f $db_struct{$Conf::Conf{'db_type'}}{$t}{$f} $options");
-			unless ($dbh->do("ALTER TABLE $t CHANGE $f $f $db_struct{$Conf::Conf{'db_type'}}{$t}{$f} $options")) {
-			    &do_log('err', 'Could not change field \'%s\' in table\'%s\'.', $f, $t);
-			    &do_log('err', 'Sympa\'s database structure may have change since last update ; please check RELEASE_NOTES');
-			    return undef;
+		if ($Conf::Conf{'update_db_field_types'} eq 'auto') {
+			if (grep $_ eq $Conf::Conf{'db_type'}, qw(SQLite mysql)) {
+				unless (&check_db_field_type(effective_format => $real_struct{$t}{$f},
+							required_format => $db_struct{$Conf::Conf{'db_type'}}{$t}{$f})) {
+					push @report, sprintf('Field \'%s\'  (table \'%s\' ; database \'%s\') does NOT have awaited type (%s). Attempting to change it...', 
+							$f, $t, $Conf::Conf{'db_name'}, $db_struct{$Conf::Conf{'db_type'}}{$t}{$f});
+					&do_log('notice', 'Field \'%s\'  (table \'%s\' ; database \'%s\') does NOT have awaited type (%s). Attempting to change it...', 
+							$f, $t, $Conf::Conf{'db_name'}, $db_struct{$Conf::Conf{'db_type'}}{$t}{$f});
+				
+					my $options;
+					if ($not_null{$f}) {
+						$options .= 'NOT NULL';
+					}
+				
+					if ($Conf::Conf{'db_type'} eq 'mysql') {
+						push @report, sprintf("ALTER TABLE $t CHANGE $f $f $db_struct{$Conf::Conf{'db_type'}}{$t}{$f} $options");
+						&do_log('notice', "ALTER TABLE $t CHANGE $f $f $db_struct{$Conf::Conf{'db_type'}}{$t}{$f} $options");
+						unless ($dbh->do("ALTER TABLE $t CHANGE $f $f $db_struct{$Conf::Conf{'db_type'}}{$t}{$f} $options")) {
+							&do_log('err', 'Could not change field \'%s\' in table\'%s\'.', $f, $t);
+							&do_log('err', 'Sympa\'s database structure may have change since last update ; please check RELEASE_NOTES');
+							return undef;
+						}
+					
+					}
+					if ($Conf::Conf{'db_type'} eq 'SQLite') {
+						my @oldfields = ();
+						foreach my $oldfield (keys %{$real_struct{$t}}) {
+							if (defined $db_struct{$Conf::Conf{'db_type'}}{${t}}{$oldfield}) {
+								push(@oldfields, $oldfield);
+							}
+						}
+						my $sqloldfields = join(", ", @oldfields);
+						my $sqlnewfieldstype = join(", ", map { "$_ $db_struct{$Conf::Conf{'db_type'}}{$t}{$_}" } keys %{$db_struct{$Conf::Conf{'db_type'}}{$t}});
+
+						# SQLite doesn't support changes on columns types hence we use a dirty workaround here:
+						# we create a temporary table (a clone of the table containing the field to update)
+						# we dump all the data into it
+						# we create a new table with the right schema
+						# we copy all the data from the temporary table into the new one
+						# we delete the temporay table
+						# see: http://www.sqlite.org/faq.html#q11
+						&do_log('notice', 'Using a dirty trick to change SQLite data type see http://www.sqlite.org/faq.html#q11');
+						my $sqlcode = "DROP TABLE IF EXISTS ${t}_backup;";
+						push @report, $sqlcode;
+						&do_log('notice', $sqlcode);
+						$dbh->do($sqlcode);
+						$sqlcode = "CREATE TEMPORARY TABLE ${t}_backup(${sqloldfields});";
+						push @report, $sqlcode;
+						&do_log('notice', $sqlcode);
+						$dbh->do($sqlcode);
+						$sqlcode = "INSERT INTO ${t}_backup SELECT ${sqloldfields} FROM ${t};";
+						push @report, $sqlcode;
+						&do_log('notice', $sqlcode);
+						$dbh->do($sqlcode);
+						$sqlcode = "DROP TABLE ${t};";
+						push @report, $sqlcode;
+						&do_log('notice', $sqlcode);
+						$dbh->do($sqlcode);
+						$sqlcode = "CREATE TABLE ${t}(${sqlnewfieldstype});";
+						push @report, $sqlcode;
+						&do_log('notice', $sqlcode);
+						$dbh->do($sqlcode);
+						$sqlcode = "INSERT INTO ${t} (${sqloldfields}) SELECT ${sqloldfields} FROM ${t}_backup;";
+						push @report, $sqlcode;
+						&do_log('notice', $sqlcode);
+						my $toto = $dbh->do($sqlcode);
+						$sqlcode = "DROP TABLE ${t}_backup;";
+						push @report, $sqlcode;
+						&do_log('notice', $sqlcode);
+						$dbh->do($sqlcode);
+						# Update real structure
+						foreach my $newfield (sort keys %{$db_struct{$Conf::Conf{'db_type'}}{$t}}) {
+							$real_struct{$t}{$newfield} = $db_struct{$Conf::Conf{'db_type'}}{$t}{$newfield};
+						}
+					}
+					push @report, sprintf('Field %s in table %s, structure updated', $f, $t);
+					&do_log('info', 'Field %s in table %s, structure updated', $f, $t);
+				}
 			}
-			
-			push @report, sprintf('Field %s in table %s, structure updated', $f, $t);
-			&do_log('info', 'Field %s in table %s, structure updated', $f, $t);
-		    }
-		}else {
-		    unless ($real_struct{$t}{$f} eq $db_struct{$Conf::Conf{'db_type'}}{$t}{$f}) {
-			&do_log('err', 'Field \'%s\'  (table \'%s\' ; database \'%s\') does NOT have awaited type (%s).', $f, $t, $Conf::Conf{'db_name'}, $db_struct{$Conf::Conf{'db_type'}}{$t}{$f});
-			&do_log('err', 'Sympa\'s database structure may have change since last update ; please check RELEASE_NOTES');
-			return undef;
-		    }
 		}
-	    }
+		}
 	    if ($Conf::Conf{'db_type'} eq 'mysql') {
 		## Check that primary key has the right structure.
 		my $should_update;
@@ -1460,7 +1525,7 @@
 		    ## drop previous index
 		    my $success;
 		    foreach my $field (@{$primary{$t}}) {
-			unless ($dbh->do("DROP INDEX $field")) {
+			unless ($dbh->do("DROP INDEX IF EXISTS $field")) {
 			    next;
 			}
 			$success = 1; last;
@@ -1474,7 +1539,7 @@
 		    }
 		    
 		    ## Add INDEX
-		    unless ($dbh->do("CREATE INDEX IF NOT EXIST $t\_index ON $t ($fields)")) {
+		    unless ($dbh->do("CREATE INDEX IF NOT EXISTS $t\_index ON $t ($fields)")) {
 			&do_log('err', 'Could not set INDEX on field \'%s\', table\'%s\'.', $fields, $t);
 			return undef;
 		    }
--- a/src/etc/script/create_db.SQLite
+++ b/src/etc/script/create_db.SQLite
@@ -18,51 +18,50 @@
 	user_subscriber		text NOT NULL,
   	custom_attribute_subscriber text,
 	robot_subscriber	text NOT NULL,
-	date_subscriber		timestamp NOT NULL,
-	update_subscriber	timestamp,
+	date_subscriber		numeric NOT NULL,
+	update_subscriber	numeric,
 	visibility_subscriber	text,
 	reception_subscriber	text,
 	topics_subscriber	text,
 	bounce_subscriber	text,
 	bounce_address_subscriber text,
 	comment_subscriber	text,
-	subscribed_subscriber 	boolean,
-	included_subscriber 	boolean,
+	subscribed_subscriber 	numeric,
+	included_subscriber 	numeric,
 	include_sources_subscriber text,
 	bounce_score_subscriber integer,
-	suspend_subscriber	boolean,
+	suspend_subscriber	numeric,
 	suspend_start_date_subscriber	integer,
 	suspend_end_date_subscriber	integer,	
 	PRIMARY KEY (robot_subscriber, list_subscriber, user_subscriber)
 );
-CREATE INDEX subscriber_idx ON subscriber_table (user_subscriber,list_subscriber,robot_subscriber);
 
 CREATE TABLE admin_table (
 	list_admin 		text NOT NULL,
  	user_admin 		text NOT NULL,
  	robot_admin 		text NOT NULL,
 	role_admin 		text NOT NULL,
-	date_admin 		timestamp NOT NULL,
-	update_admin 		timestamp,
+	date_admin 		numeric NOT NULL,
+	update_admin 		numeric,
 	reception_admin 	text,
 	visibility_admin 	text,
 	comment_admin 		text,
-	subscribed_admin  	boolean,
-	included_admin  	boolean,
+	subscribed_admin  	numeric,
+	included_admin  	numeric,
 	include_sources_admin  	text,
 	info_admin   		text,
 	profile_admin  		text,
 	PRIMARY KEY (robot_admin, list_admin, role_admin, user_admin)
 );
-CREATE	INDEX admin_idx ON admin_table(list_admin, user_admin, robot_admin, role_admin);
+CREATE INDEX admin_user_index ON admin_table ( user_admin );
 
 CREATE TABLE exclusion_table (
-	list_exclusion 		text,
- 	user_exclusion 		text,
+	list_exclusion 		text NOT NULL,
+ 	user_exclusion 		text NOT NULL,
+ 	robot_exclusion 		text NOT NULL,
 	date_exclusion 		integer,
-	PRIMARY KEY (list_exclusion, user_exclusion)
+	PRIMARY KEY (list_exclusion, user_exclusion, robot_exclusion)
 );
-CREATE	INDEX exclusion_idx ON exclusion_table(list_exclusion, user_exclusion);
 
 CREATE TABLE netidmap_table (
         netid_netidmap		text NOT NULL,
@@ -89,7 +88,6 @@
 	daemon_logs		text NOT NULL,
 	PRIMARY KEY (id_logs)					  
 );
-CREATE	INDEX logs_idx ON logs_table(id_logs);
 
 CREATE TABLE session_table (
 	id_session		text NOT NULL,
@@ -102,10 +100,9 @@
 	data_session		text,
 	PRIMARY KEY (id_session)
 );
-CREATE INDEX session_idx ON session_table(id_session);
 
 CREATE TABLE one_time_ticket_table (
-	ticket_one_time_ticket	text NOT NULL,
+	ticket_one_time_ticket	text,
 	robot_one_time_ticket	text,
 	email_one_time_ticket	text,
 	date_one_time_ticket	integer,
@@ -114,45 +111,56 @@
 	status_one_time_ticket	text,
 	PRIMARY KEY (ticket_one_time_ticket)
 );
-CREATE	INDEX one_time_ticket_idx ON one_time_ticket_table(ticket_one_time_ticket);
 
 CREATE TABLE bulkmailer_table(
-  	messagekey_bulkmailer  	varchar(80) NOT NULL,
-	packetid_bulkmailer 	varchar(33) NOT NULL,
-  	messageid_bulkmailer  	varchar(100),
+  	messagekey_bulkmailer  	text NOT NULL,
+	packetid_bulkmailer 	text NOT NULL,
+  	messageid_bulkmailer  	text,
 	receipients_bulkmailer 	text,
-	returnpath_bulkmailer 	varchar(100),
-	robot_bulkmailer 	varchar(80),
-	listname_bulkmailer 	varchar(50),
-	verp_bulkmailer 	integer,
-	merge_bulkmailer 	integer,
+	returnpath_bulkmailer 	text,
+	robot_bulkmailer 	text,
+	listname_bulkmailer 	text,
+	verp_bulkmailer 	numeric,
+	merge_bulkmailer 	numeric,
 	priority_message_bulkmailer 	integer,
 	priority_packet_bulkmailer 	integer,
 	reception_date_bulkmailer 	integer,
 	delivery_date_bulkmailer 	integer,
-	lock_bulkmailer 	varchar(30),
+	lock_bulkmailer 	text,
 	PRIMARY KEY (messagekey_bulkmailer, packetid_bulkmailer)
 );
-CREATE INDEX bulkmailer_idx ON bulkmailer_table(messagekey_bulkmailer,packetid_bulkmailer);
 
 CREATE TABLE bulkspool_table (
-  	messagekey_bulkspool  	varchar(33) NOT NULL,
-  	messageid_bulkspool  	varchar(100),
+  	messagekey_bulkspool  	text NOT NULL,
+  	messageid_bulkspool  	text,
 	message_bulkspool 	text,
-	lock_bulkspool 	integer,
-        dkim_privatekey_bulkspool  varchar(100),
-	dkim_selector_bulkspool varchar(50),
-	dkim_d_bulkspool varchar(50),
-	dkim_i_bulkspool varchar(50),
+	lock_bulkspool 	numeric,
+        dkim_privatekey_bulkspool  text,
+	dkim_selector_bulkspool text,
+	dkim_d_bulkspool text,
+	dkim_i_bulkspool text,
 	dkim_header_list_bulkspool varchar(500),
 	PRIMARY KEY (messagekey_bulkspool)
 );
-CREATE INDEX bulkspool_idx ON bulkspool_table(messagekey_bulkspool);
 
 CREATE TABLE conf_table (
-	robot_conf		text DEFAULT NULL,
-	label_conf		text DEFAULT NULL,
-	value_conf		text DEFAULT NULL,
+	robot_conf		text,
+	label_conf		text,
+	value_conf		text,
 	PRIMARY KEY (robot_conf,label_conf)
 );
-CREATE INDEX conf_idx ON conf_table(robot_conf,label_conf);
+
+CREATE TABLE list_table (
+	 creation_email_list 	text,
+	 creation_epoch_list 	numeric,
+	 editors_list 	text,
+	 name_list 	text NOT NULL,
+	 owners_list 	text,
+	 path_list 	text,
+	 robot_list 	text NOT NULL,
+	 status_list 	text,
+	 subject_list 	text,
+	 topics_list 	text,
+	 web_archive_list 	integer,
+	 PRIMARY KEY (name_list, robot_list)
+);
