Internal changes, allowing admins to set total project size limits vi…

…a admin dashboard

tests/unit/admin/test_routes.py

@@ -79,6 +79,13 @@ def test_includeme():
             traverse="/{project_name}",
             domain=warehouse,
         ),
+        pretend.call(
+            "admin.project.set_total_size_limit",
+            "/admin/projects/{project_name}/set_total_size_limit/",
+            factory="warehouse.packaging.models:ProjectFactory",
+            traverse="/{project_name}",
+            domain=warehouse,
+        ),
         pretend.call(
             "admin.project.add_role",
             "/admin/projects/{project_name}/add_role/",

tests/unit/admin/views/test_projects.py

@@ -111,6 +111,8 @@ def test_gets_project(self, db_request):
             "squattees": [squattee],
             "ONE_MB": views.ONE_MB,
             "MAX_FILESIZE": views.MAX_FILESIZE,
+            "MAX_PROJECT_SIZE": views.MAX_PROJECT_SIZE,
+            "ONE_GB": views.ONE_GB,
         }
 
     def test_non_normalized_name(self, db_request):
@@ -332,6 +334,66 @@ def test_non_normalized_name(self, db_request):
             views.journals_list(project, db_request)
 
 
+class TestProjectSetTotalSizeLimit:
+    def test_sets_total_size_limitwith_integer(self, db_request):
+        project = ProjectFactory.create(name="foo")
+
+        db_request.route_path = pretend.call_recorder(
+            lambda *a, **kw: "/admin/projects/"
+        )
+        db_request.session = pretend.stub(
+            flash=pretend.call_recorder(lambda *a, **kw: None)
+        )
+        db_request.matchdict["project_name"] = project.normalized_name
+        db_request.POST["total_size_limit"] = "150"
+
+        views.set_total_size_limit(project, db_request)
+
+        assert db_request.session.flash.calls == [
+            pretend.call("Set the total size limit on 'foo'", queue="success")
+        ]
+
+        assert project.total_size_limit == 150 * views.ONE_GB
+
+    def test_sets_total_size_limitwith_none(self, db_request):
+        project = ProjectFactory.create(name="foo")
+        project.total_size_limit = 150 * views.ONE_GB
+
+        db_request.route_path = pretend.call_recorder(
+            lambda *a, **kw: "/admin/projects/"
+        )
+        db_request.session = pretend.stub(
+            flash=pretend.call_recorder(lambda *a, **kw: None)
+        )
+        db_request.matchdict["project_name"] = project.normalized_name
+
+        views.set_total_size_limit(project, db_request)
+
+        assert db_request.session.flash.calls == [
+            pretend.call("Set the total size limit on 'foo'", queue="success")
+        ]
+
+        assert project.total_size_limit is None
+
+    def test_sets_total_size_limitwith_non_integer(self, db_request):
+        project = ProjectFactory.create(name="foo")
+
+        db_request.matchdict["project_name"] = project.normalized_name
+        db_request.POST["total_size_limit"] = "meep"
+
+        with pytest.raises(HTTPBadRequest):
+            views.set_total_size_limit(project, db_request)
+
+    def test_sets_total_size_limit_with_less_than_minimum(self, db_request):
+        project = ProjectFactory.create(name="foo")
+
+        db_request.matchdict["project_name"] = project.normalized_name
+        db_request.POST["total_size_limit"] = "90"
+
+        with pytest.raises(HTTPBadRequest):
+            views.set_total_size_limit(project, db_request)
+
+
 class TestProjectSetLimit:
     def test_sets_limitwith_integer(self, db_request):
         project = ProjectFactory.create(name="foo")

tests/unit/forklift/test_legacy.py

@@ -770,7 +770,7 @@ def test_is_duplicate_false(self, pyramid_config, db_request):
 
         assert legacy._is_duplicate_file(db_request.db, filename, wrong_hashes) is False
 
-#TODO : Project total_size test goes here
+
 class TestFileUpload:
     def test_fails_disallow_new_upload(self, pyramid_config, pyramid_request):
         pyramid_config.testing_securitypolicy(userid=1)
@@ -1880,49 +1880,6 @@ def test_upload_fails_with_too_large_file(self, pyramid_config, db_request):
             "See /the/help/url/"
         )
 
-    def test_upload_fails_with_too_large_project_size(self, pyramid_config, db_request):
-        pyramid_config.testing_securitypolicy(userid=1)
-
-        user = UserFactory.create()
-        db_request.user = user
-        EmailFactory.create(user=user)
-        project = ProjectFactory.create(
-            name="foobar", upload_limit=(60 * 1024 * 1024), # 60 MB
-            total_size = (10 * 1024 * 1024 * 1024) - 1  #10 GB
-        )
-        release = ReleaseFactory.create(project=project, version="1.0")
-        RoleFactory.create(user=user, project=project)
-
-        filename = "{}-{}.tar.gz".format(project.name, release.version)
-
-        db_request.POST = MultiDict(
-            {
-                "metadata_version": "1.2",
-                "name": project.name,
-                "version": release.version,
-                "filetype": "sdist",
-                "md5_digest": "nope!",
-                "content": pretend.stub(
-                    filename=filename,
-                    file=io.BytesIO(b"a" * (project.upload_limit + 1)),
-                    type="application/tar",
-                ),
-            }
-        )
-        db_request.help_url = pretend.call_recorder(lambda **kw: "/the/help/url/")
-
-        with pytest.raises(HTTPBadRequest) as excinfo:
-            legacy.file_upload(db_request)
-
-        resp = excinfo.value
-
-        assert db_request.help_url.calls == [pretend.call(_anchor="project-size-limit")]
-        assert resp.status_code == 400
-        assert resp.status == (
-            "400 Project size too large. Limit for project 'foobar' total size is 10 GB. "
-            "See /the/help/url/"
-        )
-
     def test_upload_fails_with_too_large_signature(self, pyramid_config, db_request):
         pyramid_config.testing_securitypolicy(userid=1)
 

warehouse/admin/routes.py

@@ -74,6 +74,13 @@ def includeme(config):
         traverse="/{project_name}",
         domain=warehouse,
     )
+    config.add_route(
+        "admin.project.set_total_size_limit",
+        "/admin/projects/{project_name}/set_total_size_limit/",
+        factory="warehouse.packaging.models:ProjectFactory",
+        traverse="/{project_name}",
+        domain=warehouse,
+    )
     config.add_route(
         "admin.project.add_role",
         "/admin/projects/{project_name}/add_role/",

warehouse/admin/templates/admin/projects/detail.html

@@ -63,6 +63,16 @@ <h4>Attributes:</h4>
                     {% endif %}
                   </td>
                 </tr>
+                <tr>
+                  <td>Total size limit</td>
+                  <td>
+                    {% if project.total_size_limit %}
+                      {{ project.total_size_limit|filesizeformat(binary=True) }}
+                    {% else %}
+                      Default ({{(MAX_PROJECT_SIZE)|filesizeformat(binary=True) }})
+                    {% endif %}
+                  </td>
+                </tr>
               </table>
             </div>
             <h4>Maintainers:</h4>
@@ -315,4 +325,36 @@ <h3 class="box-title">Set upload limit</h3>
       </div>
     </form>
   </div>
+
+ <!--Total project size limit form  -->
+  <div class="box box-secondary collapsed-box">
+    <div class="box-header with-border">
+      <h3 class="box-title">Set total size limit</h3>
+      <div class="box-tools">
+        <button class="btn btn-box-tool" data-widget="collapse" data-toggle="tooltip" title="Expand"><i class="fa fa-plus"></i></button>
+      </div>
+    </div>
+    <form method="POST" action="{{ request.route_path('admin.project.set_total_size_limit', project_name=project.normalized_name) }}">
+      <input name="csrf_token" type="hidden" value="{{ request.session.get_csrf_token() }}">
+      <div class="box-body">
+        <div class="form-group col-sm-12">
+          <label for="uploadLimit">Total size limit (in gigabytes)</label>
+          {% if project.total_size_limit %}
+            {% set total_size_limit_value = project.total_size_limit // ONE_GB %}
+          {% else %}
+            {% set total_size_limit_value = '' %}
+          {% endif %}
+          <input type="number" name="total_size_limit" class="form-control" id="totalSizeLimit" min="{{ MAX_PROJECT_SIZE // ONE_GB }}" value="{{total_size_limit_value}}">
+        </div>
+      </div>
+
+      <div class="box-footer">
+        <div class="pull-right">
+          <button type="submit" class="btn btn-primary">Submit</button>
+        </div>
+      </div>
+    </form>
+  </div>
+
+
 {% endblock %}

warehouse/admin/views/projects.py

@@ -20,12 +20,13 @@
 from sqlalchemy.orm.exc import NoResultFound
 
 from warehouse.accounts.models import User
-from warehouse.forklift.legacy import MAX_FILESIZE
+from warehouse.forklift.legacy import MAX_FILESIZE, MAX_PROJECT_SIZE
 from warehouse.packaging.models import JournalEntry, Project, Release, Role
 from warehouse.utils.paginate import paginate_url_factory
 from warehouse.utils.project import confirm_project, remove_project
 
 ONE_MB = 1024 * 1024  # bytes
+ONE_GB = 1024 * 1024 * 1024  # bytes
 
 
 @view_config(
@@ -143,6 +144,8 @@ def project_detail(project, request):
         "squattees": squattees,
         "ONE_MB": ONE_MB,
         "MAX_FILESIZE": MAX_FILESIZE,
+        "ONE_GB": ONE_GB,
+        "MAX_PROJECT_SIZE": MAX_PROJECT_SIZE,
     }
 
 
@@ -306,6 +309,47 @@ def set_upload_limit(project, request):
     )
 
 
+@view_config(
+    route_name="admin.project.set_total_size_limit",
+    permission="moderator",
+    request_method="POST",
+    uses_session=True,
+    require_methods=False,
+)
+def set_total_size_limit(project, request):
+    total_size_limit = request.POST.get("total_size_limit", "")
+
+    if not total_size_limit:
+        total_size_limit = None
+    else:
+        try:
+            total_size_limit = int(total_size_limit)
+        except ValueError:
+            raise HTTPBadRequest(
+                f"Invalid value for total size limit: {total_size_limit}, "
+                f"must be integer or empty string."
+            )
+
+        # The form is in GB, but the database field is in bytes.
+        total_size_limit *= ONE_GB
+
+        if total_size_limit < MAX_PROJECT_SIZE:
+            raise HTTPBadRequest(
+                f"Total project size can not be less than the default limit of "
+                f"{MAX_PROJECT_SIZE / ONE_GB}GB."
+            )
+
+    project.total_size_limit = total_size_limit
+
+    request.session.flash(
+        f"Set the total size limit on {project.name!r}", queue="success"
+    )
+
+    return HTTPSeeOther(
+        request.route_path("admin.project.detail", project_name=project.normalized_name)
+    )
+
+
 @view_config(
     route_name="admin.project.add_role",
     permission="admin",

warehouse/forklift/legacy.py

@@ -61,9 +61,7 @@
 
 MAX_FILESIZE = 60 * 1024 * 1024  # 60M
 MAX_SIGSIZE = 8 * 1024  # 8K
-# TODO : Finalize MAX_PROJECT_SIZE, currently 10GB
-MAX_PROJECT_SIZE = 10 * 1024 * 1024 * 1024
-
+MAX_PROJECT_SIZE = 100 * 1024 * 1024 * 1024
 PATH_HASHER = "blake2_256"
 
 
@@ -1212,18 +1210,6 @@ def file_upload(request):
                         + "See "
                         + request.help_url(_anchor="file-size-limit"),
                     )
-                # TODO : Add a check for total_size here and stop write here
-                if file_size + project.total_size > MAX_PROJECT_SIZE:
-                    raise _exc_with_message(
-                        HTTPBadRequest,
-                        "Project size too large. Limit for "
-                        + "project {name!r} total size is {limit} GB. ".format(
-                            name=project.name,
-                            limit=MAX_PROJECT_SIZE // (1024 * 1024 * 1024),
-                        )
-                        + "See "
-                        + request.help_url(_anchor="project-size-limit"),
-                    )
                 fp.write(chunk)
                 for hasher in file_hashes.values():
                     hasher.update(chunk)