submit: PostgreSQL Text type compatibility fix

* Changes Text in models because starting from SQLAlchemy>=1.0
  it arises an exception if the length is specified. (closes inveniosoftware#3037)

* Adds new column 'id' with a primary key and implements upgrade recipe
  to modify the tables sbmFORMATEXTENSION and sbmGFILERESULT.

Signed-off-by: Leonardo Rossi <[email protected]>

invenio/modules/submit/models.py

@@ -19,10 +19,10 @@
 
 """WebSubmit database models."""
 
-# General imports.
 from invenio.ext.sqlalchemy import db
 
-# Create your models here.
+from sqlalchemy.dialects import mysql
+from sqlalchemy.schema import Index
 
 
 class SbmACTION(db.Model):
@@ -135,13 +135,15 @@ class SbmCOLLECTIONSbmCOLLECTION(db.Model):
         SbmCOLLECTION,
         backref=db.backref('father', uselist=False),
         single_parent=True,
-        primaryjoin="and_(SbmCOLLECTIONSbmCOLLECTION.id_son==SbmCOLLECTION.id) "
+        primaryjoin="and_("
+        "SbmCOLLECTIONSbmCOLLECTION.id_son==SbmCOLLECTION.id) "
     )
     father = db.relationship(
         SbmCOLLECTION,
         backref=db.backref('son', uselist=False),
         single_parent=True,
-        primaryjoin="and_(SbmCOLLECTIONSbmCOLLECTION.id_father==SbmCOLLECTION.id) "
+        primaryjoin="and_("
+        "SbmCOLLECTIONSbmCOLLECTION.id_father==SbmCOLLECTION.id) "
     )
 
     @db.hybrid_property
@@ -292,18 +294,29 @@ class SbmFIELDDESC(db.Model):
     fddfi2 = db.Column(db.Text, nullable=True)
     cookie = db.Column(db.Integer(11), nullable=True,
                        server_default='0')
-    #field = db.relationship(SbmFIELD, backref='fielddescs')
+    # field = db.relationship(SbmFIELD, backref='fielddescs')
 
 
 class SbmFORMATEXTENSION(db.Model):
 
     """Represents a SbmFORMATEXTENSION record."""
 
     __tablename__ = 'sbmFORMATEXTENSION'
-    FILE_FORMAT = db.Column(db.Text(50), nullable=False,
-                            primary_key=True)
-    FILE_EXTENSION = db.Column(db.Text(10), nullable=False,
-                               primary_key=True)
+
+    id = db.Column(db.Integer(), nullable=False,
+                   primary_key=True, autoincrement=True)
+    FILE_FORMAT = db.Column(
+        db.Text().with_variant(mysql.TEXT(50), 'mysql'),
+        nullable=False)
+    FILE_EXTENSION = db.Column(
+        db.Text().with_variant(mysql.TEXT(10), 'mysql'),
+        nullable=False)
+
+
+Index('sbmformatextension_file_format_idx',
+      SbmFORMATEXTENSION.FILE_FORMAT, mysql_length=50)
+Index('sbmformatextension_file_extension_idx',
+      SbmFORMATEXTENSION.FILE_EXTENSION, mysql_length=10)
 
 
 class SbmFUNCTIONS(db.Model):
@@ -338,10 +351,19 @@ class SbmGFILERESULT(db.Model):
     """Represents a SbmGFILERESULT record."""
 
     __tablename__ = 'sbmGFILERESULT'
-    FORMAT = db.Column(db.Text(50), nullable=False,
-                       primary_key=True)
-    RESULT = db.Column(db.Text(50), nullable=False,
-                       primary_key=True)
+
+    id = db.Column(db.Integer(), nullable=False,
+                   primary_key=True, autoincrement=True)
+    FORMAT = db.Column(
+        db.Text().with_variant(db.Text(50), 'mysql'),
+        nullable=False)
+    RESULT = db.Column(
+        db.Text().with_variant(db.Text(50), 'mysql'),
+        nullable=False)
+
+
+Index('sbmgfileresult_format_idx', SbmGFILERESULT.FORMAT, mysql_length=50)
+Index('sbmgfileresult_result_idx', SbmGFILERESULT.RESULT, mysql_length=50)
 
 
 class SbmIMPLEMENT(db.Model):
@@ -480,7 +502,7 @@ class SbmSUBMISSIONS(db.Model):
                        autoincrement=True)
 
 
-__all__ = ['SbmACTION',
+__all__ = ('SbmACTION',
            'SbmALLFUNCDESCR',
            'SbmAPPROVAL',
            'SbmCATEGORIES',
@@ -503,4 +525,4 @@ class SbmSUBMISSIONS(db.Model):
            'SbmPUBLICATIONCOMM',
            'SbmPUBLICATIONDATA',
            'SbmREFEREES',
-           'SbmSUBMISSIONS']
+           'SbmSUBMISSIONS')

invenio/modules/submit/upgrades/submit_2015_04_30_fix_models_sbmFORMATEXTENSION_sbmGFILERESULT.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2015 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""Upgrade Submit models."""
+
+import warnings
+
+from invenio.ext.sqlalchemy import db
+from invenio.legacy.dbquery import run_sql
+from invenio.modules.upgrader.api import op
+
+from sqlalchemy.exc import OperationalError
+
+
+depends_on = [u'submit_2015_03_03_fix_models']
+
+
+def info():
+    """Info."""
+    return "Add a autoincrement id in sbmFORMATEXTENSION and " + \
+        "sbmGFILERESULT tables"
+
+
+def do_upgrade():
+    """Implement your upgrades here."""
+    # table sbmFORMATEXTENSION
+
+    # add "id" column
+    op.add_column('sbmFORMATEXTENSION',
+                  db.Column('id', db.Integer(), nullable=False))
+    # set all ids
+    records = run_sql("""SELECT FILE_FORMAT, FILE_EXTENSION FROM """
+                      """sbmFORMATEXTENSION AS sbm """
+                      """ORDER BY sbm.FILE_FORMAT, sbm.FILE_EXTENSION""")
+    for index, rec in enumerate(records):
+        run_sql("""UPDATE sbmFORMATEXTENSION """
+                """SET id = %s """
+                """ WHERE FILE_FORMAT = %s AND """
+                """       FILE_EXTENSION = %s """,
+                (index + 1, rec[0], rec[1]))
+    # remove primary key
+    try:
+        op.drop_constraint(None, 'sbmFORMATEXTENSION',
+                           type_='primary')
+    except OperationalError:
+        # the primary key is already dropped
+        warnings.warn("""Primary key of sbmFORMATEXTENSION """
+                      """table has been already dropped.""")
+    # set id as new primary key
+    op.create_primary_key('pk_sbmFORMATEXTENSION_id',
+                          'sbmFORMATEXTENSION', ['id'])
+    # set id as autoincrement
+    op.alter_column('sbmFORMATEXTENSION', 'id',
+                    existing_type=db.Integer(),
+                    existing_nullable=False, autoincrement=True)
+    # create indices
+    op.create_index('sbmformatextension_file_extension_idx',
+                    'sbmFORMATEXTENSION', columns=['FILE_EXTENSION'],
+                    unique=False, mysql_length=10)
+    op.create_index('sbmformatextension_file_format_idx',
+                    'sbmFORMATEXTENSION', columns=['FILE_FORMAT'],
+                    unique=False, mysql_length=50)
+
+    # table sbmGFILERESULT
+
+    # add "id" column
+    op.add_column('sbmGFILERESULT',
+                  db.Column('id', db.Integer(), nullable=False))
+    # set all ids
+    records = run_sql("""SELECT FORMAT, RESULT FROM """
+                      """sbmGFILERESULT AS sbm """
+                      """ORDER BY sbm.FORMAT, sbm.RESULT""")
+    for index, rec in enumerate(records):
+        run_sql("""UPDATE sbmGFILERESULT """
+                """SET id = %s """
+                """ WHERE FORMAT = %s AND """
+                """       RESULT = %s """,
+                (index + 1, rec[0], rec[1]))
+    # remove primary key
+    try:
+        op.drop_constraint(None, 'sbmGFILERESULT',
+                           type_='primary')
+    except OperationalError:
+        # the primary key is already dropped
+        warnings.warn("""Primary key of sbmGFILERESULT """
+                      """table has been already dropped.""")
+    # set id as new primary key
+    op.create_primary_key('pk_sbmGFILERESULT_id',
+                          'sbmGFILERESULT', ['id'])
+    # set id as autoincrement
+    op.alter_column('sbmGFILERESULT', 'id',
+                    existing_type=db.Integer(),
+                    existing_nullable=False, autoincrement=True)
+    # create indices
+    op.create_index('sbmgfileresult_format_idx',
+                    'sbmGFILERESULT', columns=['FORMAT'],
+                    unique=False, mysql_length=50)
+    op.create_index('sbmgfileresult_result_idx',
+                    'sbmGFILERESULT', columns=['RESULT'],
+                    unique=False, mysql_length=50)
+
+
+def estimate():
+    """Estimate running time of upgrade in seconds (optional)."""
+    num_sbmfe = run_sql("SELECT count(*) FROM sbmFORMATEXTENSION")
+    num_sbmgf = run_sql("SELECT count(*) FROM sbmGFILERESULT")
+    total = int(num_sbmfe[0][0]) + int(num_sbmgf[0][0])
+    return int(float(total) / 1000) + 1
+
+
+def pre_upgrade():
+    """Run pre-upgrade checks (optional)."""
+    pass
+
+
+def post_upgrade():
+    """Run post-upgrade checks (optional)."""
+    pass