diff --git a/.babelrc b/.babelrc new file mode 100644 index 000000000000..15b43f09da21 --- /dev/null +++ b/.babelrc @@ -0,0 +1,12 @@ +{ + "exclude": "node_modules/**", + "presets": [ + [ + "@babel/preset-env", + { + "useBuiltIns": "entry", + "corejs": 3 + } + ] + ] +} diff --git a/.browserslistrc b/.browserslistrc new file mode 100644 index 000000000000..4f102a5afcf9 --- /dev/null +++ b/.browserslistrc @@ -0,0 +1,2 @@ +> 0.5%, last 2 versions, Firefox ESR, not dead +not ie <= 11 diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 36826ce303d8..000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,652 +0,0 @@ -# These environment variables must be set in CircleCI UI -# -# DOCKERHUB_REPO - docker hub repo, format: / -# DOCKER_USER - Login user for docker hub -# DOCKER_PASS - Login password for docker hub user -version: 2.1 - -references: - # We declare the autograph configuration here to be able to fully leverage - # Docker executors. This configuration should be kept in sync with the - # content of `scripts/autograph_localdev_config.yaml`, which is used for - # local dev. Sadly, we cannot "include" this file here. - autograph_config: &autograph_config | - # Note: Most of the configuration here got copied from - # https://github.com/mozilla-services/autograph/blob/master/autograph.yaml - server: - # This port should be perfectly free, the upstream default of 8000 is - # used by django sometimes so let's not do that. - listen: "0.0.0.0:5500" - # cache 500k nonces to protect from authorization replay attacks - noncecachesize: 10 - - # The keys below are testing keys that do not grant any power - signers: - - id: webextensions-rsa - type: xpi - # The signing parameters for each type of add-on are 'add-on' are - # signed with the OU 'Production' and the provided ID 'extension' are - # signed with the OU 'Mozilla Extensions' and the provided ID 'system - # add-on' are signed with the OU 'Mozilla Components' and the - # provided ID - mode: add-on - recommendation: - path: "mozilla-recommendation.json" - certificate: | - -----BEGIN CERTIFICATE----- - MIIH0zCCBbugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBvDELMAkGA1UEBhMCVVMx - CzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRwwGgYDVQQKExNB - bGxpem9tIENvcnBvcmF0aW9uMSAwHgYDVQQLExdBbGxpem9tIEFNTyBEZXZlbG9w - bWVudDEYMBYGA1UEAxMPZGV2LmFtby5yb290LmNhMS4wLAYJKoZIhvcNAQkBFh9m - b3hzZWMrZGV2YW1vcm9vdGNhQG1vemlsbGEuY29tMB4XDTE3MDMyMTIzNDQwNFoX - DTI3MDMxOTIzNDQwNFowgbwxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQG - A1UEBxMNTW91bnRhaW4gVmlldzEcMBoGA1UEChMTQWxsaXpvbSBDb3Jwb3JhdGlv - bjEgMB4GA1UECxMXQWxsaXpvbSBBTU8gRGV2ZWxvcG1lbnQxGDAWBgNVBAMTD2Rl - di5hbW8ucm9vdC5jYTEuMCwGCSqGSIb3DQEJARYfZm94c2VjK2RldmFtb3Jvb3Rj - YUBtb3ppbGxhLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMdX - 5soUuvWnkVHRHN5BKByrgpuU3QioE8SNT7BwRFeqbOySdvu5ecQAdNUoRbRyFmNB - ety2rQM9qw6y8eSe9fufIgrv1sg/xj7vweLmuC8Ob+zo5/iwRQw4JUdXnDjwX3W0 - auh0QRYfxWGK3hVrP9j1zIJk/yRBornCvXTtn8C/hVSE/PWc6CuV8vTcpyj+TPni - Lvulq17NdlX5qgUdn1yougJxnznkwnoIaBYLdAyZJJIUEomiEIxfabjnh8rfSMIw - AqmslrC8F73yo4JrCqJPt1ipggfpO3ZAjlEoTMcTUgyqR8B35GyuywWR0XrkJV7N - A7BM1qNjLb2to0XQSrGyWA7uPw88LuVk2aUPDE5uNK5Kv//+SGChUn2fDZTsjj3J - KY7f39JVwh/nk8ZkApplne8fKPoknW7er2R+rejyBx1+fJjLegKQsATpgKz4LRf4 - ct34oWSV6QXrZ/KKW+frWoHncy8C+UnCC3cDBKs272yqOvBoGMQTrF5oMn8i/Rap - gBbBdwysdJXb+buf/+ZS0PUt7avKFIlXqCNZjG3xotBsTuCL5zAoVKoXJW1FwrcZ - pveQuishKWNf9Id+0HaBdDp/vlbrTwXD1zsxfYvYw8wI7NkNO3TQBni5iyG4B1wh - oR+Z5AebWuJqVnsJyjPakNiuhKNsO/xTa4TF/ymfAgMBAAGjggHcMIIB2DAPBgNV - HRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAWBgNVHSUBAf8EDDAKBggrBgEF - BQcDAzAdBgNVHQ4EFgQU2LRpqTdeQ1QlBWNA6fYAqHdpSaUwgekGA1UdIwSB4TCB - 3oAU2LRpqTdeQ1QlBWNA6fYAqHdpSaWhgcKkgb8wgbwxCzAJBgNVBAYTAlVTMQsw - CQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEcMBoGA1UEChMTQWxs - aXpvbSBDb3Jwb3JhdGlvbjEgMB4GA1UECxMXQWxsaXpvbSBBTU8gRGV2ZWxvcG1l - bnQxGDAWBgNVBAMTD2Rldi5hbW8ucm9vdC5jYTEuMCwGCSqGSIb3DQEJARYfZm94 - c2VjK2RldmFtb3Jvb3RjYUBtb3ppbGxhLmNvbYIBATBCBglghkgBhvhCAQQENRYz - aHR0cHM6Ly9jb250ZW50LXNpZ25hdHVyZS5kZXYubW96YXdzLm5ldC9jYS9jcmwu - cGVtME4GCCsGAQUFBwEBBEIwQDA+BggrBgEFBQcwAoYyaHR0cHM6Ly9jb250ZW50 - LXNpZ25hdHVyZS5kZXYubW96YXdzLm5ldC9jYS9jYS5wZW0wDQYJKoZIhvcNAQEL - BQADggIBALqVt54WTkxD5U5fHPRUSZA9rFigoIcrHNrq+gTDd057cBDUWNc0cEHV - qaP0zgzqD2bIhV/WWlfMDY3VnB8L2+Vjvu2CEt8/9Kh5x9IgBmZt5VUMuEdmQOyH - vA7lz3UI+jmUGcojtLsi+sf4kxDZh3QB3T/wGiHg+K7vXnY7GWEy1Cjfwk/dvbT2 - ODTb5B3SPGsh75VmfzFGgerzsS71LN4FYBRUklLe8ozqKF8r/jGE2vfDR1Cy09pN - oR9ti+zaBiEtMlWJjxYrv7HvuoDR9xLmPxyV6gQbo6NnbudkpNdg5LhbY3WV1IgL - TnwJ7aHXgzOZ3w/VsSctg4beZZgYnr81vLKyefWJH1VzCe5XTgwXC1R/afGiVJ0P - hA1+T4My9oTaQBsiNYA2keXKJbTKerMTupoLgV/lJjxfF5BfQiy9NL18/bzxqf+J - 7w4P/4oHt3QCdISAIhlG4ttXfRR8oz6obAb6QYdCf3x9b2/3UXKd3UJ+gwchPjj6 - InnLK8ig9scn4opVNkBkjlMRsq1yd017eQzLSirpKj3br69qyLoyb/nPNJi7bL1K - bf6m5mF5GmKR+Glvq74O8rLQZ3a75v6H+NwOqAlZnWSJmC84R2HHsHPBw+2pExJT - E5bRcttRlhEdN4NJ2vWJnOH0DENHy6TEwACINJVx6ftucfPfvOxI - -----END CERTIFICATE----- - privatekey: | - -----BEGIN PRIVATE KEY----- - MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDHV+bKFLr1p5FR - 0RzeQSgcq4KblN0IqBPEjU+wcERXqmzsknb7uXnEAHTVKEW0chZjQXrctq0DPasO - svHknvX7nyIK79bIP8Y+78Hi5rgvDm/s6Of4sEUMOCVHV5w48F91tGrodEEWH8Vh - it4Vaz/Y9cyCZP8kQaK5wr107Z/Av4VUhPz1nOgrlfL03Kco/kz54i77patezXZV - +aoFHZ9cqLoCcZ855MJ6CGgWC3QMmSSSFBKJohCMX2m454fK30jCMAKprJawvBe9 - 8qOCawqiT7dYqYIH6Tt2QI5RKEzHE1IMqkfAd+RsrssFkdF65CVezQOwTNajYy29 - raNF0EqxslgO7j8PPC7lZNmlDwxObjSuSr///khgoVJ9nw2U7I49ySmO39/SVcIf - 55PGZAKaZZ3vHyj6JJ1u3q9kfq3o8gcdfnyYy3oCkLAE6YCs+C0X+HLd+KFklekF - 62fyilvn61qB53MvAvlJwgt3AwSrNu9sqjrwaBjEE6xeaDJ/Iv0WqYAWwXcMrHSV - 2/m7n//mUtD1Le2ryhSJV6gjWYxt8aLQbE7gi+cwKFSqFyVtRcK3Gab3kLorISlj - X/SHftB2gXQ6f75W608Fw9c7MX2L2MPMCOzZDTt00AZ4uYshuAdcIaEfmeQHm1ri - alZ7Ccoz2pDYroSjbDv8U2uExf8pnwIDAQABAoICADf7eqgD3GGC1q/Yfzf3qnEq - xXo1+0EkGrEXUmrljHvmM8LYeyvEcerWifkW30SGybzENeHoN3xyhCiTnpUrAz/P - 9/qEUphYOK+SG6xCSTWF427wFb1km2+MEQQRGaFv+A8RRPjVNTYmZAM5wZbYUMz4 - cp+oB3NCL5Xll9lPpo61+pa65mN/1j/vU5TqptM/X5TJrZIke5UbNIF+pP3czNVz - 2RE4oZPbp7YnyDtwqf2jwH55vp8CcY1KemFgPGWAAWnvm7/U5Vjq6ewBSWQl9Y2R - v5bZu9fG61kRViZ6n91EksVVyOLHiNHw4LlGs0LE8a3G+6M2YQzvnHfpXLINhfwU - SZ6BWAJdknVsu6eesYoC08+nyikkq/A3BVD65pT5C9VsmUPbqqpGSYZmAuFgsf9m - zdyKVH4fOPx82DqSZEHZBojg3s5K141DmPp6o0OBX8Ydgfkg2sWXuNi/noBDvh9O - FXWN2IcgK0dET3pX4xFei0QuZgglDp3VyVVSCSUPsOwecZ2XTjtBZPCQVpp3r+QV - LyecFudQ94Ki/0R+M4CrE/mPApDvq+pTjYKFZ10YWtGIdguXq5BVZIMZfZzwIPWN - HdoaFnXRTXTlR4pLIM2nlOvyZmSMo0x6nzUMVGdv4Km9pxi6ZKAgAt4DkbCF9mt0 - QG8RpGJhiIch4kgKFmqxAoIBAQDw4X9Fp9t4f2UiessUDYxLyAtq4acu4ahup5Eb - vlDZPf9gInvz5q9aFHtYgtjTlH449f+EB4isKQscVMysgrJK+7z1IXXMm0sg44wT - F4oV+kvg3KpAridRHyE456RvCPqXYzty6ywJ9B7Zf2oCvd40JUOTm8z11K4COLut - rFIW/24PJA1CWudY/EgzD164k6379On0KryA77iKEZMUztBfHm/bdO8J/zmp7g+E - FS2TCBzR4LpN0uhBwp9wh4rVr74LrPDnQJVZKgeFd24UHEtmcVprAFNUexb2yy1s - vxnHsRPmv5eF7ED1Wlz2K+7LUWqibYOrjeCrS85+CEcey0ApAoIBAQDT2vmbHosb - Qr6ZENt6UX6n0RF8i4g3G4qhucr5hEMQs4H2J8SrUM68QT0GVY0GoDW6f79Pcyr0 - W1tm7qbAOm1Iv4uNYVL1qgpq1GnD5qpWSACGsVSE3OGELlNaVz8fgVdz6zT+rU2A - tp2t795UlrvaLgFI4wITqJF3LoTfy2MZu8JYCzlKM5pZksmEmJfR0RDAot2grtD3 - H5A+PZfUIZ/8BhmdaOAv5i647unfVF6UpPYejZ0rb67oEazxdeIHK3aD5AjurdsO - UpW/PMwsbaltp4iI7hvUfRX7Afb5fPXIhv9pHh1xWYl3djUNWmFoiMMP4tuxpOBo - y+T4maQaiDSHAoIBADrlZ9EIMclMnNXJYE4O4fbFesUvV0lHM3+ayQgXiH0Vg5Nl - 2xjPlqBX0bDajVluPU6AF3GYxfoSLv1GXqTvb9iVpKXrAHp+nef0uxMP9ltZT6Qz - UA1wh3x2OBFJ0hK0B1FsmeSHS8VDQye615jEA8iMM/GrbnnM/p7ccEcOkyO8YJSj - I/rNbzN6u8yAPZCzyx6Hy4w/xsdf1acslOHJj3kyX/cwqCGxnc/GvVR2OSZyHVnT - sLnGj7NEeudwvKlyxuzj5CMmz111wVEI2olgQa9Sl+EBu140mnDNTNYCA7OnwE3z - GoFMOrXC2mf2ZfSge4orbL5Nellnt51pOLp2x8ECggEBALM8Mazw/FOF9mbdgjJM - PFGSaa7rBcVJwdHttDHBmlPI6wzsvFEMPru6nfx76KJQbORqK9r13sN5fyzof59m - TwsbMt/cFSnOQJ39M7YPstDofbl20cDOduUzpEVsRvVKokhqGB3XVRiuZ1y+8WSz - Wh7OiTu3AwzKsrcYXkZQdnlRBq0iYcfLPKzHqUJLLzbOH9Q6djL5c8V/qLNfvNI1 - 2HqKVqV8Ex+zKJhBWRAe+x3bKnbS7MPQ6zNfsOdgCmhydwRCquPzpr7JU/PFZh+4 - b31cHgFrIZR2d2AzW1XcSLzsqa2vUs2RKOIu2deAPaUI/66zCZeTnGBNEFza76Ga - 1oUCggEAA38oXcnputwL103SeD8+uwHjtTf183Rucr+Ryqz6GymiWjlzELqu7TRd - yadAaNg9CuXmYS33Jtk/UNS0k9FvYqGTR+SBXIZr6nt9ZFd0SNlQkwkAQCsuekEs - nJlmUZax7DxXMgIHMKDboHZYM/MhgzEGSALmhU5LZ76MS17v3NEPxYpVHxjAotxW - g03HjWTltS8Bgt6u0KFTGJKEUcfwvWKZtjk5Fc1heZ49zh1nU3zo9C/h8iiijTy2 - s/YksP6cxveae4b7soN4rD/vnfsmKcG+DnTf6B8Zbm6tI2TneYOfFSCryp+yDnaJ - PIDNiTxNecePOmrD+1ivAEXcoL+e1w== - -----END PRIVATE KEY----- - - - id: webextensions-rsa-with-recommendation - type: xpi - # The signing parameters for each type of add-on are 'add-on' are - # signed with the OU 'Production' and the provided ID - # 'add-on-with-recommendation' are signed with the OU 'Production' - # and the provided ID and add a recommendation file 'extension' are - # signed with the OU 'Mozilla Extensions' and the provided ID 'system - # add-on' are signed with the OU 'Mozilla Components' and the - # provided ID 'hotfix' are signed with the OU 'Production' and the ID - # 'firefox-hotfix@mozilla.org' - mode: add-on-with-recommendation - recommendation: - path: "mozilla-recommendation.json" - states: - recommended: true - recommended-android: true - verified: true - line: true - relative_start: 0h - duration: 26298h - # RSA key gen is slow and CPU intensive, so we can optionally - # pregenerate and cache keys with a worker pool - rsacacheconfig: - numkeys: 25 - numgenerators: 2 - generatorsleepduration: 1m - fetchtimeout: 100ms - statssamplerate: 1m - certificate: | - -----BEGIN CERTIFICATE----- - MIIH0zCCBbugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBvDELMAkGA1UEBhMCVVMx - CzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRwwGgYDVQQKExNB - bGxpem9tIENvcnBvcmF0aW9uMSAwHgYDVQQLExdBbGxpem9tIEFNTyBEZXZlbG9w - bWVudDEYMBYGA1UEAxMPZGV2LmFtby5yb290LmNhMS4wLAYJKoZIhvcNAQkBFh9m - b3hzZWMrZGV2YW1vcm9vdGNhQG1vemlsbGEuY29tMB4XDTE3MDMyMTIzNDQwNFoX - DTI3MDMxOTIzNDQwNFowgbwxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQG - A1UEBxMNTW91bnRhaW4gVmlldzEcMBoGA1UEChMTQWxsaXpvbSBDb3Jwb3JhdGlv - bjEgMB4GA1UECxMXQWxsaXpvbSBBTU8gRGV2ZWxvcG1lbnQxGDAWBgNVBAMTD2Rl - di5hbW8ucm9vdC5jYTEuMCwGCSqGSIb3DQEJARYfZm94c2VjK2RldmFtb3Jvb3Rj - YUBtb3ppbGxhLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMdX - 5soUuvWnkVHRHN5BKByrgpuU3QioE8SNT7BwRFeqbOySdvu5ecQAdNUoRbRyFmNB - ety2rQM9qw6y8eSe9fufIgrv1sg/xj7vweLmuC8Ob+zo5/iwRQw4JUdXnDjwX3W0 - auh0QRYfxWGK3hVrP9j1zIJk/yRBornCvXTtn8C/hVSE/PWc6CuV8vTcpyj+TPni - Lvulq17NdlX5qgUdn1yougJxnznkwnoIaBYLdAyZJJIUEomiEIxfabjnh8rfSMIw - AqmslrC8F73yo4JrCqJPt1ipggfpO3ZAjlEoTMcTUgyqR8B35GyuywWR0XrkJV7N - A7BM1qNjLb2to0XQSrGyWA7uPw88LuVk2aUPDE5uNK5Kv//+SGChUn2fDZTsjj3J - KY7f39JVwh/nk8ZkApplne8fKPoknW7er2R+rejyBx1+fJjLegKQsATpgKz4LRf4 - ct34oWSV6QXrZ/KKW+frWoHncy8C+UnCC3cDBKs272yqOvBoGMQTrF5oMn8i/Rap - gBbBdwysdJXb+buf/+ZS0PUt7avKFIlXqCNZjG3xotBsTuCL5zAoVKoXJW1FwrcZ - pveQuishKWNf9Id+0HaBdDp/vlbrTwXD1zsxfYvYw8wI7NkNO3TQBni5iyG4B1wh - oR+Z5AebWuJqVnsJyjPakNiuhKNsO/xTa4TF/ymfAgMBAAGjggHcMIIB2DAPBgNV - HRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAWBgNVHSUBAf8EDDAKBggrBgEF - BQcDAzAdBgNVHQ4EFgQU2LRpqTdeQ1QlBWNA6fYAqHdpSaUwgekGA1UdIwSB4TCB - 3oAU2LRpqTdeQ1QlBWNA6fYAqHdpSaWhgcKkgb8wgbwxCzAJBgNVBAYTAlVTMQsw - CQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEcMBoGA1UEChMTQWxs - aXpvbSBDb3Jwb3JhdGlvbjEgMB4GA1UECxMXQWxsaXpvbSBBTU8gRGV2ZWxvcG1l - bnQxGDAWBgNVBAMTD2Rldi5hbW8ucm9vdC5jYTEuMCwGCSqGSIb3DQEJARYfZm94 - c2VjK2RldmFtb3Jvb3RjYUBtb3ppbGxhLmNvbYIBATBCBglghkgBhvhCAQQENRYz - aHR0cHM6Ly9jb250ZW50LXNpZ25hdHVyZS5kZXYubW96YXdzLm5ldC9jYS9jcmwu - cGVtME4GCCsGAQUFBwEBBEIwQDA+BggrBgEFBQcwAoYyaHR0cHM6Ly9jb250ZW50 - LXNpZ25hdHVyZS5kZXYubW96YXdzLm5ldC9jYS9jYS5wZW0wDQYJKoZIhvcNAQEL - BQADggIBALqVt54WTkxD5U5fHPRUSZA9rFigoIcrHNrq+gTDd057cBDUWNc0cEHV - qaP0zgzqD2bIhV/WWlfMDY3VnB8L2+Vjvu2CEt8/9Kh5x9IgBmZt5VUMuEdmQOyH - vA7lz3UI+jmUGcojtLsi+sf4kxDZh3QB3T/wGiHg+K7vXnY7GWEy1Cjfwk/dvbT2 - ODTb5B3SPGsh75VmfzFGgerzsS71LN4FYBRUklLe8ozqKF8r/jGE2vfDR1Cy09pN - oR9ti+zaBiEtMlWJjxYrv7HvuoDR9xLmPxyV6gQbo6NnbudkpNdg5LhbY3WV1IgL - TnwJ7aHXgzOZ3w/VsSctg4beZZgYnr81vLKyefWJH1VzCe5XTgwXC1R/afGiVJ0P - hA1+T4My9oTaQBsiNYA2keXKJbTKerMTupoLgV/lJjxfF5BfQiy9NL18/bzxqf+J - 7w4P/4oHt3QCdISAIhlG4ttXfRR8oz6obAb6QYdCf3x9b2/3UXKd3UJ+gwchPjj6 - InnLK8ig9scn4opVNkBkjlMRsq1yd017eQzLSirpKj3br69qyLoyb/nPNJi7bL1K - bf6m5mF5GmKR+Glvq74O8rLQZ3a75v6H+NwOqAlZnWSJmC84R2HHsHPBw+2pExJT - E5bRcttRlhEdN4NJ2vWJnOH0DENHy6TEwACINJVx6ftucfPfvOxI - -----END CERTIFICATE----- - privatekey: | - -----BEGIN PRIVATE KEY----- - MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDHV+bKFLr1p5FR - 0RzeQSgcq4KblN0IqBPEjU+wcERXqmzsknb7uXnEAHTVKEW0chZjQXrctq0DPasO - svHknvX7nyIK79bIP8Y+78Hi5rgvDm/s6Of4sEUMOCVHV5w48F91tGrodEEWH8Vh - it4Vaz/Y9cyCZP8kQaK5wr107Z/Av4VUhPz1nOgrlfL03Kco/kz54i77patezXZV - +aoFHZ9cqLoCcZ855MJ6CGgWC3QMmSSSFBKJohCMX2m454fK30jCMAKprJawvBe9 - 8qOCawqiT7dYqYIH6Tt2QI5RKEzHE1IMqkfAd+RsrssFkdF65CVezQOwTNajYy29 - raNF0EqxslgO7j8PPC7lZNmlDwxObjSuSr///khgoVJ9nw2U7I49ySmO39/SVcIf - 55PGZAKaZZ3vHyj6JJ1u3q9kfq3o8gcdfnyYy3oCkLAE6YCs+C0X+HLd+KFklekF - 62fyilvn61qB53MvAvlJwgt3AwSrNu9sqjrwaBjEE6xeaDJ/Iv0WqYAWwXcMrHSV - 2/m7n//mUtD1Le2ryhSJV6gjWYxt8aLQbE7gi+cwKFSqFyVtRcK3Gab3kLorISlj - X/SHftB2gXQ6f75W608Fw9c7MX2L2MPMCOzZDTt00AZ4uYshuAdcIaEfmeQHm1ri - alZ7Ccoz2pDYroSjbDv8U2uExf8pnwIDAQABAoICADf7eqgD3GGC1q/Yfzf3qnEq - xXo1+0EkGrEXUmrljHvmM8LYeyvEcerWifkW30SGybzENeHoN3xyhCiTnpUrAz/P - 9/qEUphYOK+SG6xCSTWF427wFb1km2+MEQQRGaFv+A8RRPjVNTYmZAM5wZbYUMz4 - cp+oB3NCL5Xll9lPpo61+pa65mN/1j/vU5TqptM/X5TJrZIke5UbNIF+pP3czNVz - 2RE4oZPbp7YnyDtwqf2jwH55vp8CcY1KemFgPGWAAWnvm7/U5Vjq6ewBSWQl9Y2R - v5bZu9fG61kRViZ6n91EksVVyOLHiNHw4LlGs0LE8a3G+6M2YQzvnHfpXLINhfwU - SZ6BWAJdknVsu6eesYoC08+nyikkq/A3BVD65pT5C9VsmUPbqqpGSYZmAuFgsf9m - zdyKVH4fOPx82DqSZEHZBojg3s5K141DmPp6o0OBX8Ydgfkg2sWXuNi/noBDvh9O - FXWN2IcgK0dET3pX4xFei0QuZgglDp3VyVVSCSUPsOwecZ2XTjtBZPCQVpp3r+QV - LyecFudQ94Ki/0R+M4CrE/mPApDvq+pTjYKFZ10YWtGIdguXq5BVZIMZfZzwIPWN - HdoaFnXRTXTlR4pLIM2nlOvyZmSMo0x6nzUMVGdv4Km9pxi6ZKAgAt4DkbCF9mt0 - QG8RpGJhiIch4kgKFmqxAoIBAQDw4X9Fp9t4f2UiessUDYxLyAtq4acu4ahup5Eb - vlDZPf9gInvz5q9aFHtYgtjTlH449f+EB4isKQscVMysgrJK+7z1IXXMm0sg44wT - F4oV+kvg3KpAridRHyE456RvCPqXYzty6ywJ9B7Zf2oCvd40JUOTm8z11K4COLut - rFIW/24PJA1CWudY/EgzD164k6379On0KryA77iKEZMUztBfHm/bdO8J/zmp7g+E - FS2TCBzR4LpN0uhBwp9wh4rVr74LrPDnQJVZKgeFd24UHEtmcVprAFNUexb2yy1s - vxnHsRPmv5eF7ED1Wlz2K+7LUWqibYOrjeCrS85+CEcey0ApAoIBAQDT2vmbHosb - Qr6ZENt6UX6n0RF8i4g3G4qhucr5hEMQs4H2J8SrUM68QT0GVY0GoDW6f79Pcyr0 - W1tm7qbAOm1Iv4uNYVL1qgpq1GnD5qpWSACGsVSE3OGELlNaVz8fgVdz6zT+rU2A - tp2t795UlrvaLgFI4wITqJF3LoTfy2MZu8JYCzlKM5pZksmEmJfR0RDAot2grtD3 - H5A+PZfUIZ/8BhmdaOAv5i647unfVF6UpPYejZ0rb67oEazxdeIHK3aD5AjurdsO - UpW/PMwsbaltp4iI7hvUfRX7Afb5fPXIhv9pHh1xWYl3djUNWmFoiMMP4tuxpOBo - y+T4maQaiDSHAoIBADrlZ9EIMclMnNXJYE4O4fbFesUvV0lHM3+ayQgXiH0Vg5Nl - 2xjPlqBX0bDajVluPU6AF3GYxfoSLv1GXqTvb9iVpKXrAHp+nef0uxMP9ltZT6Qz - UA1wh3x2OBFJ0hK0B1FsmeSHS8VDQye615jEA8iMM/GrbnnM/p7ccEcOkyO8YJSj - I/rNbzN6u8yAPZCzyx6Hy4w/xsdf1acslOHJj3kyX/cwqCGxnc/GvVR2OSZyHVnT - sLnGj7NEeudwvKlyxuzj5CMmz111wVEI2olgQa9Sl+EBu140mnDNTNYCA7OnwE3z - GoFMOrXC2mf2ZfSge4orbL5Nellnt51pOLp2x8ECggEBALM8Mazw/FOF9mbdgjJM - PFGSaa7rBcVJwdHttDHBmlPI6wzsvFEMPru6nfx76KJQbORqK9r13sN5fyzof59m - TwsbMt/cFSnOQJ39M7YPstDofbl20cDOduUzpEVsRvVKokhqGB3XVRiuZ1y+8WSz - Wh7OiTu3AwzKsrcYXkZQdnlRBq0iYcfLPKzHqUJLLzbOH9Q6djL5c8V/qLNfvNI1 - 2HqKVqV8Ex+zKJhBWRAe+x3bKnbS7MPQ6zNfsOdgCmhydwRCquPzpr7JU/PFZh+4 - b31cHgFrIZR2d2AzW1XcSLzsqa2vUs2RKOIu2deAPaUI/66zCZeTnGBNEFza76Ga - 1oUCggEAA38oXcnputwL103SeD8+uwHjtTf183Rucr+Ryqz6GymiWjlzELqu7TRd - yadAaNg9CuXmYS33Jtk/UNS0k9FvYqGTR+SBXIZr6nt9ZFd0SNlQkwkAQCsuekEs - nJlmUZax7DxXMgIHMKDboHZYM/MhgzEGSALmhU5LZ76MS17v3NEPxYpVHxjAotxW - g03HjWTltS8Bgt6u0KFTGJKEUcfwvWKZtjk5Fc1heZ49zh1nU3zo9C/h8iiijTy2 - s/YksP6cxveae4b7soN4rD/vnfsmKcG+DnTf6B8Zbm6tI2TneYOfFSCryp+yDnaJ - PIDNiTxNecePOmrD+1ivAEXcoL+e1w== - -----END PRIVATE KEY----- - authorizations: - - id: alice - key: fs5wgcer9qj819kfptdlp8gm227ewxnzvsuj9ztycsx08hfhzu - signers: - - webextensions-rsa - - id: bob - key: 9vh6bhlc10y63ow2k4zke7k0c3l9hpr8mo96p92jmbfqngs9e7d - signers: - - webextensions-rsa-with-recommendation - ########################################################################### - # - # The autograph configuration ends here. - # - ########################################################################### - - variables: - - &working-directory "~/addons-server" - - docker-images: - # These versions should try to match what we run in production. - # Note that the cimg/python:3.10-node convenience image provides - # nodejs latest lts, which we need to build assets and run the linter. - - &python "cimg/python:3.10-node" - - &redis "redis:6.2" - - &memcached "memcached:1.4" - - &mysql "mysql:8.0.28" - - mysql-config: &mysql-config - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: true - MYSQL_DATABASE: olympia - - defaults: &defaults - working_directory: &working-directory - docker: - - image: *python - - defaults-with-services: &defaults-with-services - <<: *defaults - docker: - - image: *python - # Below are services this project depends on. In addition to these - # services, we also need autograph and elasticsearch but not all the - # time, hence the presence of other `defaults-with-*` references. - # - # Most settings below should be kept in sync with `docker-compose.yml`. - - image: *redis - - image: *memcached - - image: *mysql - <<: *mysql-config - - defaults-with-elasticsearch: &defaults-with-elasticsearch - <<: *defaults - docker: - - image: *python - - image: *redis - - image: *memcached - - image: *mysql - <<: *mysql-config - - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3 - environment: - # Disable all xpack related features to avoid unrelated logging in - # docker logs. https://github.com/mozilla/addons-server/issues/8887 - xpack.security.enabled: false - xpack.monitoring.enabled: false - xpack.graph.enabled: false - xpack.watcher.enabled: false - discovery.type: single-node - cluster.name: default-cluster - ES_JAVA_OPTS: -Xms256m -Xmx256m - - defaults-with-autograph: &defaults-with-autograph - <<: *defaults - docker: - - image: *python - - image: *redis - - image: *memcached - - image: *mysql - <<: *mysql-config - - image: mozilla/autograph:3.3.2 - command: bash -c 'echo -e "$AUTOGRAPH_CONFIG" > amo_config.yaml && cat amo_config.yaml && /go/bin/autograph -c amo_config.yaml' - environment: - AUTOGRAPH_CONFIG: *autograph_config - - defaults-release: &defaults-release - machine: - image: ubuntu-2004:202201-02 - working_directory: &working-directory - -commands: - make_release: - description: "Builds and pushes a Docker image" - parameters: - image_tag: - type: string - default: "latest" - steps: - - run: > - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' - "$CIRCLE_SHA1" - "$CIRCLE_TAG" - "$CIRCLE_PROJECT_USERNAME" - "$CIRCLE_PROJECT_REPONAME" - "$CIRCLE_BUILD_URL" - > version.json - - run: - name: Build docker image and push to repo - command: | - docker version - docker login -u "${DOCKERHUB_USER}" -p "${DOCKERHUB_PASS}" - docker build -t app:build --label git.commit="$CIRCLE_SHA1" . - docker tag app:build "${DOCKERHUB_REPO}":<< parameters.image_tag >> - docker push "${DOCKERHUB_REPO}":<< parameters.image_tag >> - - better_checkout: - description: circle ci checkout step on steroids - parameters: - clone_options: - type: string - default: --depth=1 - description: git clone options - fetch_options: - type: string - default: --depth=10 - description: git fetch options - steps: - - run: - name: checkout - command: | - #!/bin/sh - set -e - - # Workaround old docker images with incorrect $HOME - # check https://github.com/docker/docker/issues/2968 for details - if [ "${HOME}" = "/" ] - then - export HOME=$(getent passwd $(id -un) | cut -d: -f6) - fi - - export SSH_CONFIG_DIR="$HOME/.ssh" - - echo "Using SSH Config Dir $SSH_CONFIG_DIR" - - mkdir -p "$SSH_CONFIG_DIR" - - echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=' >> "$SSH_CONFIG_DIR/known_hosts" - - (umask 077; touch "$SSH_CONFIG_DIR/id_rsa") - chmod 0600 "$SSH_CONFIG_DIR/id_rsa" - (cat $CHECKOUT_KEY > "$SSH_CONFIG_DIR/id_rsa") - - export GIT_SSH_COMMAND='ssh -i $SSH_CONFIG_DIR/id_rsa -o UserKnownHostsFile=$SSH_CONFIG_DIR/known_hosts' - - # use git+ssh instead of https - git config --global url."ssh://git@github.com".insteadOf "https://github.com" || true - git config --global gc.auto 0 || true - - if [ -e .git ] - then - git remote set-url origin "$CIRCLE_REPOSITORY_URL" || true - else - git clone << parameters.clone_options >> "$CIRCLE_REPOSITORY_URL" . - fi - - if [ -n "$CIRCLE_TAG" ] - then - git fetch << parameters.fetch_options >> --force origin "refs/tags/${CIRCLE_TAG}" - elif [[ "$CIRCLE_BRANCH" =~ ^pull\/* ]] - then - git fetch << parameters.fetch_options >> --force origin "${CIRCLE_BRANCH}/head:remotes/origin/${CIRCLE_BRANCH}" - else - git fetch << parameters.fetch_options >> --force origin "${CIRCLE_BRANCH}:remotes/origin/${CIRCLE_BRANCH}" - fi - - - if [ -n "$CIRCLE_TAG" ] - then - git reset --hard "$CIRCLE_SHA1" - git checkout -q "$CIRCLE_TAG" - elif [ -n "$CIRCLE_BRANCH" ] - then - git reset --hard "$CIRCLE_SHA1" - git checkout -q -B "$CIRCLE_BRANCH" - fi - - git reset --hard "$CIRCLE_SHA1" - - setup_container: - description: common setup for the primary container - parameters: - wait_services: - type: boolean - default: true - steps: - - better_checkout - - run: - name: Initial setup - command: | - curl -sL https://repo.mysql.com/mysql-apt-config_0.8.29-1_all.deb --output mysql-apt-config.deb - sudo dpkg -i mysql-apt-config.deb - sudo apt-get update -q - sudo apt-get install -y gettext pngcrush librsvg2-bin libmysqlclient-dev - sudo cp ./docker/etc/mime.types /etc/mime.types - sudo touch /addons-server-docker-container - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz - rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz - environment: - DOCKERIZE_VERSION: v0.6.1 - - when: - condition: << parameters.wait_services >> - steps: - - run: - name: Wait for redis - command: dockerize -wait tcp://localhost:6379 -timeout 1m - - run: - name: Wait for mysql - command: dockerize -wait tcp://localhost:3306 -timeout 1m - - run: - name: Wait for memcached - command: dockerize -wait tcp://localhost:11211 -timeout 1m - - run: - name: Set environment variables - command: | - echo export CPUCOUNT=2 >> $BASH_ENV - echo export NPM_CONFIG_PREFIX=/deps/ >> $BASH_ENV - echo export CC=\"`python -c 'import sysconfig; print(sysconfig.get_config_var("CC"))'`\" >> $BASH_ENV - cat $BASH_ENV - - run: - name: Install Python and Node dependencies - command: | - sudo mkdir /deps - sudo chown circleci /deps - ACTUAL_CIRCLE_WORKING_DIRECTORY="${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}" - ln -s ${ACTUAL_CIRCLE_WORKING_DIRECTORY}/package.json /deps/package.json - ln -s ${ACTUAL_CIRCLE_WORKING_DIRECTORY}/package-lock.json /deps/package-lock.json - make update_deps - # should be executed after all python install commands - - run: pyenv rehash - -jobs: - addons-versions-files-ratings: - <<: *defaults-with-services - steps: - - setup_container - - run: - command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/addons/ src/olympia/versions/ src/olympia/files/ src/olympia/ratings/ - - amo-lib-locales-and-signing: - <<: *defaults-with-autograph - steps: - - setup_container - - run: - command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/amo/ src/olympia/lib/ src/olympia/signing - environment: - AUTOGRAPH_SERVER_URL: http://127.0.0.1:5500 - # After having removed `tox`, we noticed that this job was a lot slower. - # The reason seems to be related to pyenv shims, which add a huge - # overhead to CLIs like `dennis-cmd`. By updating the path below, we - # should use `dennis-cmd` directly instead of the pyenv shim. - - run: echo "export PATH=\"$(pyenv prefix)/bin:$PATH\"" >> $BASH_ENV - - run: bash ./locale/compile-mo.sh ./locale/ - - run: - command: pytest -n 2 -m 'needs_locales_compilation' -v src/olympia/ - - assets: - <<: *defaults-with-services - steps: - - setup_container - - run: make update_assets - - run: - command: pytest -m "static_assets" -v src/olympia/ - - run: make run_js_tests - - codestyle: - <<: *defaults - steps: - - setup_container: - wait_services: false - - run: pyenv rehash - - run: make lint-codestyle - - localization: - <<: *defaults - steps: - - setup_container: - wait_services: false - - run : make extract_locales - - run: git diff - - devhub: - <<: *defaults-with-services - steps: - - setup_container - - run: - command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/devhub/ - - docs: - <<: *defaults - steps: - - setup_container: - wait_services: false - - run: make docs - - main: - <<: *defaults-with-services - steps: - - setup_container - - run: - command: | - pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets and not internal_routes_allowed' \ - -v src/olympia/ \ - --ignore src/olympia/addons/ \ - --ignore src/olympia/devhub/ \ - --ignore src/olympia/files/ \ - --ignore src/olympia/reviewers/ \ - --ignore src/olympia/ratings/ \ - --ignore src/olympia/amo/ \ - --ignore src/olympia/lib/ \ - --ignore src/olympia/signing \ - --ignore src/olympia/versions/ \ - --ignore src/olympia/zadmin - - reviewers-and-zadmin: - <<: *defaults-with-autograph - steps: - - setup_container - - run: - command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/reviewers/ src/olympia/zadmin/ - environment: - AUTOGRAPH_SERVER_URL: http://127.0.0.1:5500 - - run: - name: internal_routes_allowed - command: | - # We need to change the setting in the file because we can't - # override an env variable here, and the next command requires - # `INTERNAL_ROUTES_ALLOWED` to be set to `True`. - sed -i 's/^INTERNAL_ROUTES_ALLOWED.*/INTERNAL_ROUTES_ALLOWED=True/' settings_test.py - pytest -m 'internal_routes_allowed' -v src/olympia/ - - es-tests: - <<: *defaults-with-elasticsearch - steps: - - setup_container - - run: - name: Wait for elasticsearch - command: dockerize -wait tcp://localhost:9200 -timeout 1m - - run: - command: pytest -m "es_tests and not needs_locales_compilation and not static_assets" -v src/olympia/ - - release-master: - <<: *defaults-release - steps: - - checkout - - make_release: - image_tag: latest - - release-tag: - <<: *defaults-release - steps: - - checkout - - make_release: - image_tag: "${CIRCLE_TAG}" - -workflows: - version: 2 - default-workflow: - jobs: - - addons-versions-files-ratings - - amo-lib-locales-and-signing - - assets - - codestyle - - devhub - - docs - - main - - reviewers-and-zadmin - - es-tests - - localization - - release-master: - filters: - branches: - only: master - tags: - ignore: /.*/ - - release-tag: - filters: - tags: - only: /.*/ - branches: - ignore: /.*/ diff --git a/.dockerignore b/.dockerignore index 3327bb406839..a8e2271f7e87 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,69 @@ +# Copy of .gitignore please keep these in sync +*.css.tmp +*.egg-info +*.js.tmp +*.less.css +*.mo +*.po~ +*.py[co] +*.signed.zip +*.styl.css +*-all.css +*-all.js +*-min.css +*-min.js +.*.sw? +.cache +.DS_Store .env +.ipython +.mysql_history +.nose* +.npm/ +.pdbhistory +.pytest_cache +.ssh +.tox/ +.vscode +backups +build*.py +buildx-bake-metadata.json +deps/* +docker*.yml +docker/artifacts/* +docs/_build +docs/_gh-pages +docs/api/_build +local_settings.py +MANIFEST +node_modules +pip-log.txt +private/ +.ruff_cache +settings_local.py* +settings_local_*.py +shellng_local.py +site-static/* +src/olympia/discovery/strings.jinja2 +static-build/* +static/css/node_lib/* +static/js/node_lib/* +storage +tmp/* + +# Additionally ignore these files from the docker build that are not in .gitignore + +.dockerignore +.github +docs +private +docker-bake.hcl +docker-compose*.yml +Dockerfile* +Makefile-os + +# Do not copy .git directory to docker images .git -.cache -.tox -deps/ -node_modules/ -storage/ -logs/* + +# Ignore generated credentials from google-github-actions/auth +gha-creds-*.json diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 000000000000..2091a55b7d4a --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,29 @@ +(contributing)= + +# Contributing + +If you're not sure this is the correct place to file an issue +then please file an issue on the [mozilla/addons] project instead. + +Before contributing code, please note: + +- You agree to license your contributions under the [license]. +- Please ask on the [dev-addons mailing list] before submitting + pull-requests for new features or large changes that are not related to + existing issues. +- Follow [PEP8], [jshint] and our other [style guide conventions]. +- Please write tests and read the docs on [addons-server]. + +Ready to get started? Follow [these steps](https://wiki.mozilla.org/Add-ons/Contribute/Code). + +**Note to staff:** If you come across a potential "good first bug" for contributors, please tag it with "**maybe good first bug**". The community team [triages](https://wiki.mozilla.org/Add-ons/Contribute/Goodfirstbugs_triage) these every other week to ensure they have mentors assigned, onboarding information, and basic steps to get started. This gives new contributors a better experience when they pick a "good first bug" to work on. + +Thank you for contributing! + +[addons-server]: https://addons-server.readthedocs.io/en/latest/ +[dev-addons mailing list]: https://mail.mozilla.org/listinfo/dev-addons +[jshint]: http://www.jshint.com/ +[license]: https://github.com/mozilla/addons-server/blob/master/LICENSE +[mozilla/addons]: https://github.com/mozilla/addons/issues/new +[pep8]: https://www.python.org/dev/peps/pep-0008/ +[style guide conventions]: https://mozweb.readthedocs.io/en/latest/ diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst deleted file mode 100644 index 5b40638cbccb..000000000000 --- a/.github/CONTRIBUTING.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. _contributing: - -============ -Contributing -============ - -If you're not sure this is the correct place to file an issue -then please file an issue on the `mozilla/addons`_ project instead. - -Before contributing code, please note: - -- You agree to license your contributions under the `license`_. -- Please ask on the `dev-addons mailing list`_ before submitting - pull-requests for new features or large changes that are not related to - existing issues. -- Follow `PEP8`_, `jshint`_ and our other `style guide conventions`_. -- Please write tests and read the docs on `addons-server`_. - -Ready to get started? Follow `these steps `_. - -**Note to staff:** If you come across a potential "good first bug" for contributors, please tag it with "**maybe good first bug**". The community team `triages `_ these every other week to ensure they have mentors assigned, onboarding information, and basic steps to get started. This gives new contributors a better experience when they pick a "good first bug" to work on. - -Thank you for contributing! - -.. _license: https://github.com/mozilla/addons-server/blob/master/LICENSE -.. _dev-addons mailing list: https://mail.mozilla.org/listinfo/dev-addons -.. _PEP8: https://www.python.org/dev/peps/pep-0008/ -.. _jshint: http://www.jshint.com/ -.. _style guide conventions: https://mozweb.readthedocs.io/en/latest/ -.. _addons-server: https://addons-server.readthedocs.io/en/latest/ -.. _mozilla/addons: https://github.com/mozilla/addons/issues/new diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 09fc8fde6e4e..4906acfcc999 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,15 +1,43 @@ +Fixes: mozilla/addons#ISSUENUM + + + +### Description + + + +### Context + + + +### Testing + + + +### Checklist + -* [ ] This PR relates to an existing open issue and there are no existing - PRs open for the same issue. -* [ ] Add `Fixes #ISSUENUM` at the top of your PR. -* [ ] Add a description of the changes introduced in this PR. -* [ ] The change has been successfully run locally. -* [ ] Add tests to cover the changes added in this PR. -* [ ] Add before and after screenshots (Only for changes that impact the UI). - -Once you have met the above requirements please replace this section with -a `Fixes #ISSUENUM` linking to the issue fixed by this PR along with an -explanation of the changes. Thanks for your contribution! +- [ ] Add `#ISSUENUM` at the top of your PR to an existing open issue in the mozilla/addons repository. +- [ ] Successfully verified the change locally. +- [ ] The change is covered by automated tests, or otherwise indicated why doing so is unnecessary/impossible. +- [ ] Add before and after screenshots (Only for changes that impact the UI). +- [ ] Add or update relevant [docs](../docs/) reflecting the changes made. diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml new file mode 100644 index 000000000000..8e728e1a2dd2 --- /dev/null +++ b/.github/actions/build-docker/action.yml @@ -0,0 +1,84 @@ +name: 'Docker Build Action' +description: 'Build the docker image' +inputs: + registry: + required: true + description: The registry to tag the image with + image: + required: true + description: The image to tag the image with + version: + required: true + description: The image version to tag with + push: + required: false + description: Push the image? + default: 'false' + +outputs: + tag: + description: The docker tag of the built image + value: ${{ steps.build_meta.outputs.tag }} + version: + description: The docker version of the built image + value: ${{ steps.meta.outputs.version }} + digest: + description: The docker build digest of the built image + value: ${{ steps.build_meta.outputs.digest }} + +runs: + using: 'composite' + steps: + - name: Context + id: context + shell: bash + run: | + git_repo_url="${{ github.server_url }}/${{ github.repository }}" + + echo "git_build_url=$git_repo_url/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT + echo "git_sha=${{ github.sha }}" >> $GITHUB_OUTPUT + echo "metadata_file=buildx-bake-metadata.json" >> $GITHUB_OUTPUT + + cat $GITHUB_OUTPUT + + - name: Docker meta + id: meta + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 + with: + bake-target: web + images: ${{ inputs.registry }}/${{ inputs.image }} + tags: | + # use raw tag to allow the calling workflow to define the version of the image + # and to prevent multiple tags from being associated with a build + type=raw,value=${{ inputs.version }} + + - name: Build Image + id: build + shell: bash + env: + # These values are irrelevant for local builds, so don't include them + # in the make command directly but set them on the environment to be picked up. + DOCKER_TAGS_FILE: ${{ steps.meta.outputs.bake-file-tags }} + DOCKER_ANNOTATIONS_FILE: ${{ steps.meta.outputs.bake-file-annotations }} + DOCKER_METADATA_FILE: ${{ steps.context.outputs.metadata_file }} + DOCKER_PUSH: ${{ inputs.push }} + version: ${{ steps.meta.outputs.version }} + git_sha: ${{ steps.context.outputs.git_sha }} + git_build_url: ${{ steps.context.outputs.git_build_url }} + run: | + make build \ + DOCKER_VERSION="$version" \ + DOCKER_COMMIT="$git_sha" \ + DOCKER_BUILD="$git_build_url" + + - name: Get image digest + id: build_meta + shell: bash + env: + metadata_file: ${{ steps.context.outputs.metadata_file }} + run: | + metadata=$(cat $metadata_file) + echo "digest=$(echo $metadata | jq -r '.web."containerimage.digest"')" >> $GITHUB_OUTPUT + echo "tag=$(echo $metadata | jq -r '.web."image.name"')" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT + diff --git a/.github/actions/run-docker/action.yml b/.github/actions/run-docker/action.yml new file mode 100644 index 000000000000..eaf00e7b8573 --- /dev/null +++ b/.github/actions/run-docker/action.yml @@ -0,0 +1,72 @@ +name: 'Docker Run Action' +description: 'Run a command in a new container' +inputs: + version: + description: 'The version of the image to run. ' + required: true + default: 'local' + digest: + description: 'The build digest of the image to run. Overrides version.' + required: true + default: '' + run: + description: 'Run command in container' + required: true + logs: + description: 'Show logs' + required: false + skip_data_seed: + description: 'Skip data backup' + required: false + default: 'true' + target: + description: 'Docker target to run (development|production)' + required: false + default: 'production' + deps: + description: 'Which dependencies to install at runtime? (development|production)' + required: false + default: 'production' + +runs: + using: 'composite' + steps: + - name: Run Docker Container + id: run + continue-on-error: true + shell: bash + env: + version: ${{ inputs.version }} + digest: ${{ inputs.digest }} + target: ${{ inputs.target }} + deps: ${{ inputs.deps }} + skip_data_seed: ${{ inputs.skip_data_seed }} + run: ${{ inputs.run }} + run: | + # Start the specified services + make up \ + DOCKER_VERSION="${version}" \ + DOCKER_DIGEST="${digest}" \ + DOCKER_TARGET="${target}" \ + OLYMPIA_UID="$(id -u)" \ + OLYMPIA_DEPS="${deps}" \ + SKIP_DATA_SEED="${skip_data_seed}" \ + DOCKER_WAIT="true" + + docker compose exec --user olympia web bash -c "${run}" + + - name: Post Run (logs and exit code) + if: always() + shell: bash + env: + logs: ${{ inputs.logs }} + outcome: ${{ steps.run.outcome }} + run: | + if [[ "$logs" == "true" ]]; then + docker compose logs + fi + + # If the run command failed, exit with a non-zero exit code + if [ "$outcome" != 'success' ]; then + exit 1 + fi diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9653b409031c..1cf2a5e451d3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,15 +1,51 @@ version: 2 updates: +- package-ecosystem: "docker" + directory: "/" + schedule: + interval: weekly + cooldown: + default-days: 7 + open-pull-requests-limit: 99 +- package-ecosystem: "docker-compose" + directory: "/" + schedule: + interval: weekly + cooldown: + default-days: 7 + open-pull-requests-limit: 99 +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: weekly + cooldown: + default-days: 7 + open-pull-requests-limit: 99 - package-ecosystem: "npm" directory: "/" schedule: interval: daily open-pull-requests-limit: 99 + cooldown: + default-days: 7 + exclude: + - addons-linter - package-ecosystem: pip directory: "/requirements" schedule: interval: daily + groups: + google: + patterns: + - "google-*" + celery: + patterns: + - celery + - billiard + - kombu open-pull-requests-limit: 99 + cooldown: + default-days: 7 ignore: - dependency-name: django versions: @@ -34,7 +70,7 @@ updates: - ">= 6" - dependency-name: idna versions: - - ">= 3" + - ">= 4" - dependency-name: amqp versions: - ">= 6" diff --git a/.github/release-template.md b/.github/release-template.md new file mode 100644 index 000000000000..76d17362f3d1 --- /dev/null +++ b/.github/release-template.md @@ -0,0 +1,28 @@ +This week's push hero is @{{GITHUB_USER}} + +Previous Release: [{{PREVIOUS_TAG}}]({{PREVIOUS_RELEASE_URL}}) + +## Blockers: + +## Cherry-picks: + + + +## Before we push: + +## Before we start: + +## Before we promote: + +## After we're done: + +## Addons-Frontend Changelog: + + + +## Addons Server Changelog: + + diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 000000000000..f0633633c47d --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,11 @@ +changelog: + categories: + - title: Notable things shipping + labels: + - '*' + exclude: + labels: + - dependencies + - title: Dependendabots + labels: + - dependencies diff --git a/.github/workflows/_health_check.yml b/.github/workflows/_health_check.yml new file mode 100644 index 000000000000..a13492fac364 --- /dev/null +++ b/.github/workflows/_health_check.yml @@ -0,0 +1,171 @@ +name: Health check + +run-name: | + ref: ${{ github.ref_name }} | + environment: ${{ inputs.environment }} | + +permissions: {} + +on: + workflow_call: + inputs: + environment: + description: The environment to run the health check in + type: string + required: false + notify: + description: Whether to notify the user of the health check outcome + type: boolean + required: false + slack_channel: + description: The Slack channel to send notifications to + type: string + required: false + secrets: + SLACK_TOKEN: + description: The Slack token to use for notifications + required: false + + workflow_dispatch: + inputs: + environment: + description: The environment to run the health check in + type: choice + required: true + options: + - prod + - stage + - dev + - host + notify: + description: Whether to notify the user of the health check outcome + type: boolean + required: false + slack_channel: + description: The Slack channel to send notifications to + type: string + required: false + +env: + health_check_file: health_check.json + health_check_blocks_file: health_check_blocks.json + +jobs: + context: + runs-on: ubuntu-latest + outputs: + slack_channel: ${{ steps.environment.outputs.slack_channel }} + notify: ${{ steps.environment.outputs.notify }} + environment: ${{ steps.environment.outputs.environment }} + + steps: + - name: Environment + id: environment + env: + notify: ${{ inputs.notify }} + environment: ${{ inputs.environment }} + slack_channel: ${{ inputs.slack_channel }} + run: | + if [[ "$environment" == "prod" ]]; then + environment="prod" + elif [[ "$environment" == "stage" ]]; then + environment="stage" + elif [[ "$environment" == "dev" ]]; then + environment="dev" + else + environment="host" + fi + + { + echo "notify=$notify" + echo "slack_channel=$slack_channel" + echo "environment=$environment" + } >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + health_check: + runs-on: ubuntu-latest + needs: [context] + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - uses: actions/setup-python@v6 + with: + python-version: 3.12 + + - name: Set waffle switch dummy-monitor-fails to true + if: needs.context.outputs.environment == 'host' + uses: ./.github/actions/run-docker + with: + version: local + run: | + ./manage.py waffle_switch dummy-monitor-fails on + + - name: Run health check + continue-on-error: true + id: health_check + shell: bash + env: + environment: ${{ needs.context.outputs.environment }} + run: | + ./scripts/health_check.py \ + --env "${environment}" \ + --verbose \ + --output "${health_check_file}" + + - name: Set message blocks + id: blocks + if: steps.health_check.outcome == 'failure' + shell: bash + run: | + if [ ! -f "$health_check_file" ]; then + echo "Health check file not found" + exit 1 + fi + + # Create the message blocks file + ./scripts/health_check_blocks.py \ + --input "$health_check_file" \ + --output "${{ env.health_check_blocks_file }}" + + # Multiline output needs to use a delimiter to be passed to + # the GITHUB_OUTPUT file. + blocks=$(cat "${{ env.health_check_blocks_file }}") + echo "blocks<> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + - name: Notify Failure + if: steps.blocks.outcome == 'success' && needs.context.outputs.slack_channel != '' + uses: mozilla/addons/.github/actions/slack@f748fdc2c8057ad7b574521a60c182939f2ee687 + with: + slack_token: ${{ secrets.SLACK_TOKEN }} + payload: | + { + "channel": "${{ needs.context.outputs.slack_channel }}", + "blocks": ${{ toJson(steps.blocks.outputs.blocks) }}, + "text": "[${{ needs.context.outputs.environment }}] Health check ${{ steps.health_check.outcome }}", + # Don't unfurl links or media + "unfurl_links": false, + "unfurl_media": false, + } + dry_run: ${{ !(needs.context.outputs.environment != 'host' && needs.context.outputs.notify == 'true') }} + + - name: Exit with outcome + shell: bash + env: + environment: ${{ needs.context.outputs.environment }} + outcome: ${{ steps.health_check.outcome }} + run: | + + if [[ "$environment" == 'host' && "$outcome" != "failure" ]]; then + echo "Health check should fail on host environment to simulate a real failure" + exit 1 + fi + + if [[ "$environment" != 'host' && "$outcome" == "failure" ]]; then + echo "Health check should not fail on a live environment, that is a real failure!" + exit 1 + fi diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml new file mode 100644 index 000000000000..df3d2f820408 --- /dev/null +++ b/.github/workflows/_test.yml @@ -0,0 +1,68 @@ +name: Test Docker Image + +run-name: | + ref: ${{ github.ref_name }} | + version: ${{ inputs.version }} | + digest: ${{ inputs.digest }} | + +permissions: {} + +on: + workflow_call: + inputs: + version: + description: The version of the image to run + type: string + required: true + digest: + description: The build digest of the image to run. Overrides version. + type: string + required: false + workflow_dispatch: + inputs: + version: + description: The version of the image to run + type: string + required: true + digest: + description: The build digest of the image to run. Overrides version. + type: string + required: false + +concurrency: + group: test-${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}}-${{ toJson(inputs) }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - + name: Needs Locale Compilation + run: | + make compile_locales + make test_needs_locales_compilation + - + name: Internal Routes + run: make test_internal_routes_allowed + - + name: Elastic Search + run: make test_es_tests + - + name: Codestyle + run: make lint-codestyle + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Test (${{ matrix.name }}) + uses: ./.github/actions/run-docker + with: + version: ${{ inputs.version }} + digest: ${{ inputs.digest }} + deps: development + run: ${{ matrix.run }} diff --git a/.github/workflows/_test_check.yml b/.github/workflows/_test_check.yml new file mode 100644 index 000000000000..42de0253ba86 --- /dev/null +++ b/.github/workflows/_test_check.yml @@ -0,0 +1,188 @@ +name: Test make up and check the local dev setup + +run-name: | + ref: ${{ github.ref_name }} | + version: ${{ inputs.version }} | + digest: ${{ inputs.digest }} | + +permissions: {} + +on: + workflow_call: + inputs: + version: + description: The version of the image to run + type: string + required: true + digest: + description: The build digest of the image to run. Overrides version. + type: string + required: false + workflow_dispatch: + inputs: + version: + description: The version of the image to run + type: string + required: true + digest: + description: The build digest of the image to run. Overrides version. + type: string + required: false + +concurrency: + group: test_check-${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}}-${{ toJson(inputs) }} + cancel-in-progress: true + +jobs: + context: + runs-on: ubuntu-latest + outputs: + is_fork: ${{ steps.context.outputs.is_fork }} + steps: + - id: context + uses: mozilla/addons/.github/actions/context@f748fdc2c8057ad7b574521a60c182939f2ee687 + + test_check: + runs-on: ubuntu-latest + name: | + version: '${{ matrix.version }}' | + target: '${{ matrix.target }}' | + deps: '${{ matrix.deps }}' + strategy: + fail-fast: false + matrix: + version: + - local + - ${{ inputs.version }} + target: + - development + - production + deps: + - development + - production + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: ${{ matrix.version == 'local' && 'Uncached Build' || 'Pull' }} Check + id: build + continue-on-error: true + uses: ./.github/actions/run-docker + # Set environment variables that are expected to be ignored + env: + DOCKER_COMMIT: 'not-expected' + DOCKER_VERSION: 'not-expected' + with: + version: ${{ matrix.version }} + target: ${{ matrix.target }} + deps: ${{ matrix.deps }} + run: make check + + - name: Verify build + id: verify + shell: bash + env: + version: ${{ matrix.version }} + target: ${{ matrix.target }} + outcome: ${{ steps.build.outcome }} + run: | + if [[ "$version" != "local" && "$target" == "development" && "$outcome" == "success" ]]; then + echo "Non local images must run in production mode" + exit 1 + fi + + - name: Cached Build Check + uses: ./.github/actions/run-docker + if: matrix.version == 'local' && steps.verify.outcome == 'success' + with: + version: ${{ matrix.version }} + target: ${{ matrix.target }} + deps: ${{ matrix.deps }} + run: echo true + + test_make_docker_configuration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - uses: actions/setup-node@v6 + with: + node-version: 20 + package-manager-cache: false + - name: Install dependencies + shell: bash + run: npm ci + - name: Check make/docker configuration + shell: bash + run: | + docker compose version + make test_setup + - name: Test setup + uses: ./.github/actions/run-docker + with: + digest: ${{ inputs.digest }} + version: ${{ inputs.version }} + deps: development + run: | + pytest tests/make/ + + test_verify_failure: + runs-on: ubuntu-latest + needs: context + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Create failure + id: failure + continue-on-error: true + uses: ./.github/actions/run-docker + with: + digest: ${{ inputs.digest }} + version: ${{ inputs.version }} + run: | + exit 1 + + - name: Verify failure + if: always() + env: + outcome: ${{ steps.failure.outcome }} + run: | + if [[ "$outcome" != "failure" ]]; then + echo "Expected failure" + exit 1 + fi + + test_special_characters: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Check (special characters in command) + uses: ./.github/actions/run-docker + with: + digest: ${{ inputs.digest }} + version: ${{ inputs.version }} + run: | + echo 'this is a question?' + echo 'a * is born' + echo 'wow an array []' + + test_migrations: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - uses: ./.github/actions/run-docker + with: + version: ${{ inputs.version }} + run: echo true diff --git a/.github/workflows/_test_main.yml b/.github/workflows/_test_main.yml new file mode 100644 index 000000000000..dcb423f0bace --- /dev/null +++ b/.github/workflows/_test_main.yml @@ -0,0 +1,145 @@ +name: Test Docker Image (test_main only) + +run-name: | + ref: ${{ github.ref_name }} | + version: ${{ inputs.version }} | + digest: ${{ inputs.digest }} | + splits: ${{ inputs.splits }} | + +permissions: {} + +on: + workflow_call: + inputs: + version: + description: The version of the image to run + type: string + required: false + digest: + description: The build digest of the image to run. Overrides version. + type: string + required: false + splits: + description: How many splits for test_main + type: number + required: false + default: 14 + workflow_dispatch: + inputs: + version: + description: The version of the image to run + type: string + required: true + digest: + description: The build digest of the image to run. Overrides version. + type: string + required: false + splits: + description: How many splits for test_main + type: number + required: false + default: 14 + +concurrency: + group: test_main-${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}}-${{ toJson(inputs) }} + cancel-in-progress: true + +env: + log_artifact: test_main_logs + log_file: report.json + +jobs: + test_config: + runs-on: ubuntu-latest + + outputs: + matrix: ${{ steps.result.outputs.matrix }} + splits: ${{ steps.result.outputs.splits }} + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Calculate splits + id: result + shell: bash + env: + splits: ${{ inputs.splits }} + run: | + # the matrix group should be indexed from 1 to splits + 1 + n=$((splits + 1)) + matrix=$(jq -n -c --arg n "$n" '[range(1;($n|tonumber))]') + + # validate that matrix is an array of numbers from 1 to splits + 1 + if ! echo "$matrix" | jq empty; then + echo "matrix is not an array of numbers from 1 to splits + 1" + exit 1 + fi + + echo "splits=$splits" >> "$GITHUB_OUTPUT" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + test_main: + runs-on: ubuntu-latest + needs: [test_config] + strategy: + fail-fast: false + matrix: + group: ${{fromJson(needs.test_config.outputs.matrix)}} + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Test (test_matrix) + uses: ./.github/actions/run-docker + with: + digest: ${{ inputs.digest }} + version: ${{ inputs.version }} + deps: development + run: | + split="--splits ${{ needs.test_config.outputs.splits }}" + group="--group ${{ matrix.group }}" + report="--report-log ${{ env.log_file}}" + make test_main ARGS="${split} ${group} ${report}" + + - name: Upload logs + uses: actions/upload-artifact@v5 + with: + path: ${{ env.log_file }} + name: ${{ env.log_artifact }}-${{ matrix.group }} + retention-days: 1 + overwrite: true + + test_log: + runs-on: ubuntu-latest + if: always() + needs: [test_config, test_main] + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - uses: actions/download-artifact@v6 + with: + pattern: ${{ env.log_artifact }}* + + - name: Cat logs + shell: bash + run: | + for dir in $(find . -maxdepth 1 -name "${{ env.log_artifact }}*" -type d | sort -V); do + job=$(basename "$dir") + file="${dir}/${{ env.log_file }}" + if [ -f "$file" ]; then + jq \ + -r \ + --arg job "$job" \ + 'select(has("when") and .when == "teardown") | "[\($job)] \(.outcome) \(.nodeid)"' < "$file" + else + echo "$file: No such file or directory" + fi + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..1c506291945e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,364 @@ +name: CI + +on: + # Runs when there is a push to the default branch + # This triggers tests and a pushed "latest" image + # That is deployed to the "dev" environment + push: + branches: + - master + # Runs on pull requests to verify changes and push + # PR image for local testing + pull_request: + # Manually dispatch run entire CI on a ref + workflow_dispatch: + # Runs when a release is published + # Pushes a tagged image + # That is deployed to the "staging/production" environments + release: + types: [published] + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}} + cancel-in-progress: true + +permissions: {} + +env: + docs_artifact: docs + +jobs: + context: + runs-on: ubuntu-latest + + outputs: + is_fork: ${{ steps.context.outputs.is_fork }} + is_release_master: ${{ steps.context.outputs.is_release_master }} + is_default_branch: ${{ steps.context.outputs.is_default_branch }} + is_release_tag: ${{ steps.context.outputs.is_release_tag }} + docker_version: ${{ steps.context.outputs.docker_version }} + + git_ref: ${{ steps.git.outputs.git_ref }} + git_ref_link: ${{ steps.git.outputs.git_ref_link }} + + steps: + - name: Set context + id: context + uses: mozilla/addons/.github/actions/context@f748fdc2c8057ad7b574521a60c182939f2ee687 + + - name: Git Reference + id: git + env: + sha: ${{ github.sha }} + event_name: ${{ github.event_name }} + pr_title: ${{ github.event.pull_request.title }} + pr_link: ${{ github.event.pull_request.html_url}} + push_title: ${{ github.event.head_commit.message }} + push_link: ${{ github.event.head_commit.url }} + release_title: ${{ github.event.release.name }} + release_link: ${{ github.event.release.html_url }} + git_url: ${{ github.server_url }}/${{ github.repository }} + run: | + short_sha=$(echo "${sha}" | cut -c1-7) + + git_ref="no ref" + git_ref_link="${git_url}" + + if [[ "${event_name}" == "pull_request" ]]; then + git_ref="${pr_title}" + git_ref_link="${pr_link}" + elif [[ "${event_name}" == "push" ]]; then + git_ref="${short_sha} (${push_title}...)" + git_ref_link="${push_link}" + elif [[ "${event_name}" == "release" ]]; then + git_ref="${release_title}" + git_ref_link="${release_link}" + fi + + git_ref=$(echo -e "${git_ref}" | head -n1 | tr -dc '[:print:]') + echo "git_ref=${git_ref}" >> "$GITHUB_OUTPUT" + echo "git_ref_link=${git_ref_link}" >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + test_actions: + permissions: + contents: 'read' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: [actionlint, zizmor] + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Run ${{ matrix.target }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: make ${{ matrix.target }} + + build: + name: ${{ needs.context.outputs.is_fork == 'true' && 'Skip' || 'Build' }} CI Image + runs-on: ubuntu-latest + needs: context + + outputs: + # If build is skipped we should pass local version to build the image + version: ${{ steps.build.outputs.version || 'local' }} + digest: ${{ steps.build.outputs.digest || '' }} + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Login to Dockerhub + if: needs.context.outputs.is_fork == 'false' + id: docker_hub + uses: mozilla/addons/.github/actions/login-docker@f748fdc2c8057ad7b574521a60c182939f2ee687 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + + - name: Build and Push Image + if: steps.docker_hub.outcome == 'success' + id: build + uses: ./.github/actions/build-docker + with: + registry: ${{ steps.docker_hub.outputs.registry }} + image: ${{ steps.docker_hub.outputs.image }} + version: ci-${{ needs.context.outputs.docker_version }} + push: true + + docs_build: + runs-on: ubuntu-latest + needs: build + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - uses: actions/configure-pages@v5 + + - name: Build Docs + uses: ./.github/actions/run-docker + with: + digest: ${{ needs.build.outputs.digest }} + version: ${{ needs.build.outputs.version }} + deps: development + run: | + make docs + + - name: Upload artifact + uses: actions/upload-pages-artifact@v4 + with: + path: 'docs/_build/html' + name: ${{ env.docs_artifact }} + + docs_deploy: + needs: [context, docs_build] + # Only deploy docs on a push event + # to the default branch + # that is not running on a fork + if: | + github.event_name == 'push' && + needs.context.outputs.is_default_branch == 'true' && + needs.context.outputs.is_fork == 'false' + permissions: + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + with: + artifact_name: ${{ env.docs_artifact }} + + locales: + runs-on: ubuntu-latest + needs: [build, context] + permissions: + contents: write + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: ${{ needs.context.outputs.is_default_branch }} + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + + - name: Extract Locales + uses: ./.github/actions/run-docker + with: + digest: ${{ needs.build.outputs.digest }} + version: ${{ needs.build.outputs.version }} + deps: development + run: make extract_locales + + - name: Push Locales + if: | + github.event_name == 'push' || + github.event_name == 'pull_request' + shell: bash + env: + is_fork: ${{ needs.context.outputs.is_fork }} + is_default_branch: ${{ needs.context.outputs.is_default_branch }} + is_push: ${{ github.event_name == 'push' }} + run: | + if [[ "$is_fork" == 'true' ]]; then + cat <<'EOF' + Github actions are not authorized to push from workflows triggered by forks. + We cannot verify if the l10n extraction push will work or not. + Please submit a PR from the base repository if you are modifying l10n extraction scripts. + EOF + else + if [[ "$is_default_branch" == 'true' && "$is_push" == 'true' ]]; then + args="" + else + args="--dry-run" + fi + make push_locales ARGS="${args}" + fi + + health_check: + needs: context + uses: ./.github/workflows/_health_check.yml + with: + environment: host + slack_channel: ${{ vars.SLACK_ADDONS_PRODUCTION_CHANNEL }} + + test: + needs: build + uses: ./.github/workflows/_test.yml + with: + version: ${{ needs.build.outputs.version }} + digest: ${{ needs.build.outputs.digest }} + + test_main: + needs: [context, build] + uses: ./.github/workflows/_test_main.yml + with: + version: ${{ needs.build.outputs.version }} + digest: ${{ needs.build.outputs.digest }} + + test_check: + needs: [context, build] + uses: ./.github/workflows/_test_check.yml + with: + version: ${{ needs.build.outputs.version }} + digest: ${{ needs.build.outputs.digest }} + + push_dockerhub: + name: Push Production Docker Image (Dockerhub) + runs-on: ubuntu-latest + if: | + needs.context.outputs.is_release_master == 'true' || + needs.context.outputs.is_release_tag == 'true' + needs: [context, build, docs_build, locales, test, test_main] + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Login to Dockerhub + id: docker_hub + uses: mozilla/addons/.github/actions/login-docker@f748fdc2c8057ad7b574521a60c182939f2ee687 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + + - name: Build and Push Image + id: build + uses: ./.github/actions/build-docker + with: + registry: ${{ steps.docker_hub.outputs.registry }} + image: ${{ steps.docker_hub.outputs.image }} + version: ${{ needs.context.outputs.docker_version }} + push: true + + push_gar: + name: Push Production Docker Image (GAR) + runs-on: ubuntu-latest + if: | + needs.context.outputs.is_release_master == 'true' || + needs.context.outputs.is_release_tag == 'true' + needs: [context, build, docs_build, locales, test, test_main] + + permissions: + contents: 'read' + id-token: 'write' + + env: + registry: us-docker.pkg.dev + image: moz-fx-amo-prod/amo-prod/addons-server + + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Login to GAR + id: docker_gar + uses: mozilla/addons/.github/actions/login-gar@f748fdc2c8057ad7b574521a60c182939f2ee687 + with: + registry: ${{ env.registry }} + service_account: ${{ secrets.GAR_PUSHER_SERVICE_ACCOUNT_EMAIL }} + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + + - name: Build and Push Image + id: build + uses: ./.github/actions/build-docker + with: + registry: ${{ env.registry }} + image: ${{ env.image }} + version: ${{ needs.context.outputs.docker_version }} + push: true + + notification: + runs-on: ubuntu-latest + name: Notification (${{ matrix.name }}) + if: needs.context.outputs.is_fork == 'false' && always() + needs: [context, push_dockerhub, push_gar] + strategy: + matrix: + include: + - + name: Master Build + dry_run: ${{ needs.context.outputs.is_release_master != 'true' }} + - + name: Release Build + dry_run: ${{ needs.context.outputs.is_release_tag != 'true' }} + + steps: + + - name: Slack Notification + uses: mozilla/addons/.github/actions/slack-workflow-notification@f748fdc2c8057ad7b574521a60c182939f2ee687 + with: + slack_token: ${{ secrets.SLACK_TOKEN }} + slack_channel: ${{ vars.SLACK_ADDONS_PRODUCTION_CHANNEL }} + conclusion: "${{ needs.push_gar.result }}" + text: "${{ needs.context.outputs.git_ref }}" + text_link: "${{ needs.context.outputs.git_ref_link }}" + context: | + { + "actor": "${{ github.event.sender.login }}", + "event": "${{ github.event_name }}", + "env": "ci", + "repo": "${{ github.repository }}" + } + links: | + { + "${{ github.run_id }}": "${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }}", + "${{ github.repository }}": "${{ github.server_url }}/${{ github.repository }}" + } + dry_run: ${{ matrix.dry_run }} diff --git a/.github/workflows/draft_release.yml b/.github/workflows/draft_release.yml new file mode 100644 index 000000000000..a9e30d34c588 --- /dev/null +++ b/.github/workflows/draft_release.yml @@ -0,0 +1,91 @@ +name: Draft Release + +on: + workflow_dispatch: + inputs: + push_hero: + description: The person responsible for facilitating the release. + required: true + type: choice + options: + - diox + - eviljeff + tag: + description: 'Release date YYYY.MM.DD Also used to generate the tag name.' + required: true + +permissions: {} + +jobs: + draft_release: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Create Release Draft + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.event.inputs.tag }} + push_hero: ${{ github.event.inputs.push_hero }} + event_name: ${{ github.event_name }} + shell: bash + run: | + if [ -z "$tag" ]; then + echo "Tag is required" + exit 1 + fi + + if [ -z "$push_hero" ]; then + echo "Push hero is required" + exit 1 + fi + + # Validate the tag is formatted correctly YYYY.MM.DD or YYYY.MM.DD-X + # where X is a whole number greater than zero + if [[ ! $tag =~ ^[0-9]{4}\.[0-9]{2}\.[0-9]{2}(-[0-9]+)?$ ]]; then + echo "Invalid tag format. Must be YYYY.MM.DD or YYYY.MM.DD-X" + exit 1 + fi + + # Verify that a release with this tag does not already exist + if gh release view "$tag" &> /dev/null; then + echo "Release $tag-next already exists" + exit 1 + fi + + # Get the latest release tag + previous_release=$( + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/mozilla/addons-server/releases/latest + ) + + previous_tag=$(echo "$previous_release" | jq -r '.tag_name') + previous_release_url=$(echo "$previous_release" | jq -r '.html_url') + + cat < + template=${template//\{\{GITHUB_USER\}\}/$push_hero} + + # Replace {{PREVIOUS_TAG}} in template with $previous_tag + template=${template//\{\{PREVIOUS_TAG\}\}/$previous_tag} + + # Replace {{PREVIOUS_RELEASE_URL}} in template with $previous_release_url + template=${template//\{\{PREVIOUS_RELEASE_URL\}\}/$previous_release_url} + + gh release create "$tag" \ + --title "$tag" \ + --target master \ + --notes "$template" \ + --draft diff --git a/.github/workflows/health_check.yml b/.github/workflows/health_check.yml new file mode 100644 index 000000000000..aa024e2fbb45 --- /dev/null +++ b/.github/workflows/health_check.yml @@ -0,0 +1,100 @@ +name: Health Check + +on: + repository_dispatch: + workflow_dispatch: + schedule: + # Every 5 minutes + - cron: '*/5 * * * *' + +permissions: {} + +env: + health_check_file: health_check.json + health_check_blocks_file: health_check_blocks.json + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + context: + runs-on: ubuntu-latest + outputs: + is_fork: ${{ steps.context.outputs.is_fork }} + steps: + - name: Context + id: context + uses: mozilla/addons/.github/actions/context@f748fdc2c8057ad7b574521a60c182939f2ee687 + + health_check: + strategy: + fail-fast: false + matrix: + environment: + - dev + - stage + - prod + + uses: ./.github/workflows/_health_check.yml + secrets: + SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} + with: + environment: ${{ matrix.environment }} + notify: ${{ github.event_name == 'schedule' }} + slack_channel: ${{ vars.SLACK_ADDONS_PRODUCTION_CHANNEL }} + + post_health_check: + if: always() + needs: [context, health_check] + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Retry Health Check + if: needs.health_check.result == 'failure' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ref: ${{ github.ref }} + run: | + sleep 10 + gh workflow run health_check.yml --ref "${ref}" + + - name: Notify Recovery + uses: mozilla/addons/.github/actions/slack@f748fdc2c8057ad7b574521a60c182939f2ee687 + with: + slack_token: ${{ secrets.SLACK_TOKEN }} + payload: | + { + "channel": "${{ vars.SLACK_ADDONS_PRODUCTION_CHANNEL }}", + "text": ":white_check_mark: Health check has recovered", + } + dry_run: ${{ !(needs.health_check.result == 'success' && github.event_name == 'workflow_dispatch') }} + + - name: Notify Failure + uses: mozilla/addons/.github/actions/slack-workflow-notification@f748fdc2c8057ad7b574521a60c182939f2ee687 + with: + slack_token: ${{ secrets.SLACK_TOKEN }} + slack_channel: ${{ vars.SLACK_ADDONS_PRODUCTION_CHANNEL }} + conclusion: "failure" + text: ${{ github.ref }} + text_link: ${{ github.event.repository.html_url }} + context: | + { + "channel": "${{ vars.SLACK_ADDONS_PRODUCTION_CHANNEL }}", + "text": ":x: Health check has failed" + } + links: | + { + "${{ github.run_id }}": "${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }}", + "${{ github.repository }}": "${{ github.server_url }}/${{ github.repository }}" + } + dry_run: ${{ !(github.event_name == 'schedule' && needs.health_check.result == 'failure') }} + + + diff --git a/.gitignore b/.gitignore index 9d17895c5637..ef4a66ef9077 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,3 @@ -*-all.css -*-all.js -*-min.css -*-min.js *.css.tmp *.egg-info *.js.tmp @@ -11,6 +7,10 @@ *.py[co] *.signed.zip *.styl.css +*-all.css +*-all.js +*-min.css +*-min.js .*.sw? .cache .DS_Store @@ -20,42 +20,40 @@ .nose* .npm/ .pdbhistory -.pytest_cache/* +.pytest_cache +.ssh .tox/ .vscode +backups build*.py +buildx-bake-metadata.json +deps/* docker*.yml docker/artifacts/* +docs/_build docs/_gh-pages docs/api/_build -docs/_build +docs/topics/models local_settings.py -logs/* MANIFEST node_modules pip-log.txt +private/ +.ruff_cache settings_local.py* settings_local_*.py shellng_local.py site-static/* src/olympia/discovery/strings.jinja2 +static-build/* static/css/node_lib/* -static/js/i18n/*.js static/js/node_lib/* -storage/files/* -storage/git-storage/* -storage/mlbf/* -storage/guarded-addons/* -storage/shared_storage/* -storage/sitemaps/* -supervisord.pid +storage/* tmp/* -deps/* -# private projects -private/ +# End of .gitignore. Please keep this in sync with the top section of .dockerignore # do not ignore the following files !docker-compose.private.yml +!docker-compose.tools.yml !private/README.md -!deps/.keep diff --git a/.prettierignore b/.prettierignore deleted file mode 100644 index b742a87c3a97..000000000000 --- a/.prettierignore +++ /dev/null @@ -1,30 +0,0 @@ -# exclude everything by default -*.* - -# exclude these files -Dockerfile* -LICENSE -Makefile* - -# exclude these directories -.pytest_cache/ -.tox/ -/docker/ -/docs/ -/private/ -/scripts/ -/site-static/ -/src/ -/storage/ -__pycache__/ -i18n/ -img/ -node_lib/ -deps/ - -# safe-list files we want to process -!*.js - -# exclude generated JS files -*-all.js -*-min.js diff --git a/.prettierrc b/.prettierrc deleted file mode 100644 index 3c49d5bc49dc..000000000000 --- a/.prettierrc +++ /dev/null @@ -1,6 +0,0 @@ -{ - "arrowParens": "always", - "singleQuote": true, - "trailingComma": "all", - "proseWrap": "never" -} diff --git a/.python-version b/.python-version new file mode 100644 index 000000000000..6995277bbf94 --- /dev/null +++ b/.python-version @@ -0,0 +1,3 @@ +# dependabot uses this file to determine which python version to use, see +# https://github.com/dependabot/dependabot-core/issues/1455 +3.12 diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 11b5f9811691..000000000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,19 +0,0 @@ -# .readthedocs.yaml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -build: - os: ubuntu-20.04 - tools: - python: "3.10" - -# Build documentation in the docs/ directory with Sphinx -sphinx: - configuration: docs/conf.py - -python: - install: - - requirements: requirements/docs.txt diff --git a/Dockerfile b/Dockerfile index 7e12814f44ce..81d70efc2c1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,105 +1,207 @@ -FROM python:3.10-slim-buster +##### Important information for maintaining this Dockerfile ######################################## +# Read docs/topics/development/building_and_running_services.md for more info about this Dockerfile. +#################################################################################################### -# Should change it to use ARG instead of ENV for OLYMPIA_UID/OLYMPIA_GID -# once the jenkins server is upgraded to support docker >= v1.9.0 -ENV OLYMPIA_UID=9500 \ - OLYMPIA_GID=9500 -RUN groupadd -g ${OLYMPIA_GID} olympia && useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_GID} -s /sbin/nologin -d /data/olympia olympia +FROM python:3.13-slim-bookworm@sha256:50b530b93da5a3ce61c6b7eabdb1a90f15d509ca1f026d9ab7cb2cb04f4dda46 AS olympia -# Add support for https apt repos and gpg signed repos -RUN apt-get update && apt-get install -y \ - apt-transport-https \ - gnupg2 \ - && rm -rf /var/lib/apt/lists/* +ENV BUILD_INFO=/build-info.json +ENV ENV=build + +# Set shell to bash with logs and errors for build +SHELL ["/bin/bash", "-xue", "-c"] + +ENV OLYMPIA_UID=9500 +# give olympia access to the HOME directory +ENV HOME=/data/olympia +ENV DEPS_DIR=${HOME}/deps +ENV NPM_DEPS_DIR=${HOME}/node_modules +ENV VITE_MANIFEST_FILE_NAME=manifest.json +ENV STATIC_URL_PREFIX=/static-server/ +ENV MEDIA_URL_PREFIX=/user-media/ +ENV ELASTIC_CLIENT_APIVERSIONING=1 +RUN < ${BUILD_INFO} +{ + "commit": "${DOCKER_COMMIT}", + "version": "${DOCKER_VERSION}", + "build": "${DOCKER_BUILD}", + "target": "${DOCKER_TARGET}", + "source": "https://github.com/mozilla/addons-server" +} +INNEREOF +# Set permissions to make the file readable by all but only writable by root +chmod 644 ${BUILD_INFO} +EOF + +FROM olympia AS base # Add keys and repos for node and mysql -COPY docker/*.gpg.key /etc/pki/gpg/ -RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \ - apt-key add /etc/pki/gpg/nodesource.gpg.key \ - && APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \ - apt-key add /etc/pki/gpg/mysql.gpg.key +# TODO: replace this with a bind mount on the RUN command +COPY docker/*.gpg.asc /etc/apt/trusted.gpg.d/ COPY docker/*.list /etc/apt/sources.list.d/ -# Allow scripts to detect we're running in our own container and install -# packages. -RUN touch /addons-server-docker-container \ - && apt-get update && apt-get install -y \ - # General (dev-) dependencies - bash-completion \ - build-essential \ - curl \ - libjpeg-dev \ - libsasl2-dev \ - libxml2-dev \ - libxslt-dev \ - locales \ - zlib1g-dev \ - libffi-dev \ - libssl-dev \ - nodejs \ - # Git, because we're using git-checkout dependencies - git \ - # Dependencies for mysql-python (from mysql apt repo, not debian) - mysql-client \ - libmysqlclient-dev \ - swig \ - gettext \ - # Use rsvg-convert to render our static theme previews - librsvg2-bin \ - # Use pngcrush to optimize the PNGs uploaded by developers - pngcrush \ - && rm -rf /var/lib/apt/lists/* +RUN < settings_local.py && DJANGO_SETTINGS_MODULE='settings_local' locale/compile-mo.sh locale \ - && DJANGO_SETTINGS_MODULE='settings_local' python manage.py compress_assets \ - && DJANGO_SETTINGS_MODULE='settings_local' python manage.py generate_jsi18n_files \ - && DJANGO_SETTINGS_MODULE='settings_local' python manage.py collectstatic --noinput \ - && npm prune --production \ - && ./scripts/generate_build.py > build.py \ - && rm -f settings_local.py settings_local.pyc +FROM base AS development + +# Copy build info from info +COPY --from=info ${BUILD_INFO} ${BUILD_INFO} + +FROM base AS locales +ARG LOCALE_DIR=${HOME}/locale +# Compile locales +# Copy the locale files from the host so it is writable by the olympia user +COPY --chown=olympia:olympia locale ${LOCALE_DIR} +# Copy the executable individually to improve the cache validity +RUN \ + --mount=type=bind,source=requirements/locale.txt,target=${HOME}/requirements/locale.txt \ + --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \ + --mount=type=bind,source=scripts/compile_locales.py,target=${HOME}/scripts/compile_locales.py \ + make -f Makefile-docker compile_locales + +# More efficient caching by mounting the exact files we need +# and copying only the static/ & locale/ directory. +FROM pip_production AS assets + +# In order to create js i18n files with all of our strings, we need to include +# the compiled locale files +COPY --from=locales --chown=olympia:olympia ${HOME}/locale/ ${HOME}/locale/ +# TODO: only copy the files we need for compiling assets +COPY --chown=olympia:olympia static/ ${HOME}/static/ + +# Finalize the build +# TODO: We should move update_assets to the `builder` stage once we can efficiently +# Run that command without having to copy the whole source code +# This will shave nearly 1 minute off the best case build time +RUN \ + --mount=type=bind,src=src,target=${HOME}/src \ + --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \ + --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \ + --mount=type=bind,src=manage.py,target=${HOME}/manage.py \ + --mount=type=bind,src=package.json,target=${HOME}/package.json \ + --mount=type=bind,src=package-lock.json,target=${HOME}/package-lock.json \ + --mount=type=bind,src=vite.config.ts,target=${HOME}/vite.config.ts \ +< .env - -.PHONY: initialize_docker -initialize_docker: create_env_file -# Run a fresh container from the base image to install deps. Since /deps is -# shared via a volume in docker-compose.yml, this installs deps for both web -# and worker containers, and does so without requiring the containers to be up. -# We just create dummy empty package.json and package-lock.json in deps/ so -# that docker compose doesn't create dummy ones itself, as they would be owned -# by root. They don't matter: the ones at the root directory are mounted -# instead. - touch deps/package.json - touch deps/package-lock.json -# Note that this is running with --user ${UID}:${GID} because the user olympia -# would be uid 9500 regardless of host at this point (this is only fixed when -# the container is up, through the command defined in docker-compose.yml), -# which is wrong for local development. - docker compose run --rm --user ${UID}:${GID} web make update_deps - docker compose up -d - docker compose exec --user olympia web make initialize +.PHONY: docker_compose_config +docker_compose_config: ## Show the docker compose configuration + @docker compose config web --format json + +.PHONY: docker_build_web +docker_build_web: ## Build the docker images using buildx bake + docker buildx bake $(DOCKER_BAKE_ARGS) $(ARGS) + +.PHONY: docker_pull_web +docker_pull_web: ## Pull the latest docker image using current tag + docker compose pull web --policy always + +.PHONY: docker_pull_or_build ## Pull or build the docker image based on the image version +docker_pull_or_build: +# If the image is tagged with version "local" then we should build the image before running +# docker compose up. The image will be available to docker compose, skipping a pull attempt. +# This is useful for local development where the image is built and tagged with "local". +# Also for CI/CID pipelines on forks where we cannot pull the image and must build locally. +# If the image is tagged with a version other than "local" then we should skip the build +# and let docker compose pull the image instead. This is useful for CI/CD pipelines where +# the image is already built and pushed to a registry. + @IMAGE=$$(docker compose config web --format json | jq -r '.services.web.image'); \ + echo "image: $$IMAGE"; \ + if echo "$$IMAGE" | grep -q ":local"; then \ + $(MAKE) docker_build_web; \ + else \ + $(MAKE) docker_pull_web; \ + fi + +.PHONY: docker_mysqld_volume_create +docker_mysqld_volume_create: ## Create the mysqld volume + docker volume create $(DOCKER_MYSQLD_VOLUME) + +.PHONY: docker_mysqld_volume_remove +docker_mysqld_volume_remove: ## Remove the mysqld volume + docker volume rm $(DOCKER_MYSQLD_VOLUME) + +.PHONY: docker_compose_down +docker_compose_down: ## Stop the docker containers + docker compose down --rmi local --remove-orphans --volumes + +.PHONY: docker_clean_volumes +docker_clean_volumes: ## Remove dangling volumes, skipping the mysqld volume + docker volume prune \ + --filter label=com.docker.compose.project=addons-server \ + --all \ + --force + +.PHONY: docker_clean_images +docker_clean_images: ## Remove dangling images + docker image prune --filter "dangling=true" --force + +.PHONY: docker_clean_build_cache +docker_clean_build_cache: ## Remove buildx build cache + docker buildx prune -af + +.PHONY: clean_docker +clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache ## Remove all docker resources taking space on the host machine + +.PHONY: docker_sync_host +docker_sync_host: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target + docker compose run \ + --rm \ + --no-deps \ + $(DOCKER_RUN_ARGS) \ + web \ + ./scripts/sync_host_files.py + +.PHONY: up_pre +up_pre: setup docker_pull_or_build docker_sync_host ## Pre-up the environment, setup files, volumes and host state + +.PHONY: up_start +up_start: docker_mysqld_volume_create ## Start the docker containers + docker compose $(DOCKER_COMPOSE_ARGS) up $(DOCKER_COMPOSE_UP_ARGS) $(ARGS) + +.PHONY: up_post +up_post: docker_clean_images docker_clean_volumes ## Post-up the environment, setup files, volumes and host state +# Explicitly run initialize via the web container as make can get confused +# both routing the command to the web container and +# routing the command to the proper target. + docker compose exec --user olympia web make -f Makefile-docker initialize ARGS=$(shell echo "'$(INITIALIZE_ARGS)'") + +.PHONY: up +up: up_pre up_start up_post ## Up the environment + +.PHONY: up_private +up_private: DOCKER_COMPOSE_ARGS += -f docker-compose.yml -f docker-compose.private.yml +up_private: up ## Up the environment with the private services + +.PHONY: down +down: docker_compose_down docker_clean_images docker_clean_volumes ## Stop the docker containers and clean up non-peristent dangling resources %: ## This directs any other recipe (command) to the web container's make. - docker compose exec --user olympia web make $(MAKECMDGOALS) ARGS=$(ARGS) + docker compose exec --user olympia web make $(MAKECMDGOALS) ARGS="$(shell echo $(ARGS))" # You probably want to put new commands in Makefile-docker, unless they operate # on multiple containers or are host-os specific. diff --git a/README.md b/README.md new file mode 100644 index 000000000000..ff8d201247f4 --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +[![Code of conduct](https://img.shields.io/badge/%E2%9D%A4-code%20of%20conduct-blue.svg)](https://github.com/mozilla/addons-server/blob/master/.github/CODE_OF_CONDUCT.md) + +[![CI Status](https://github.com/mozilla/addons-server/actions/workflows/ci.yml/badge.svg)](https://github.com/mozilla/addons-server/actions/workflows/ci.yml) + +# Addons-Server + +Welcome to the Addons Server repository! Please feel free to visit the web page of the current project hosted on [addons.mozilla.org]. If you want to develop or run the applciation locally, please follow the [setup and configuration][setup_link] docs. We'd love your help! You can come talk to us on [Matrix #amo:mozilla.org] if you have any questions. + +Please report bugs here: + +You can access the AMO dev environment at and the AMO stage environment at + +% marker-for-security-bug-inclusion-do-not-remove + +# Security Bug Reports + +This code and its associated production web page are included in the Mozilla’s web and services [bug bounty program]. If you find a security vulnerability, please submit it via the process outlined in the program and [FAQ pages]. Further technical details about this application are available from the [Bug Bounty Onramp page]. + +Please submit all security-related bugs through Bugzilla using the [web security bug form]. Never submit security-related bugs through a Github Issue or by email. + +[addons.mozilla.org]: https://addons.mozilla.org +[bug bounty onramp page]: https://wiki.mozilla.org/Security/BugBountyOnramp/ +[bug bounty program]: https://www.mozilla.org/en-US/security/web-bug-bounty/ +[faq pages]: https://www.mozilla.org/en-US/security/bug-bounty/faq-webapp/ +[setup_link]: https://mozilla.github.io/addons-server/topics/development/setup_and_configuration.html +[matrix #amo:mozilla.org]: https://chat.mozilla.org/#/room/#amo:mozilla.org +[web security bug form]: https://bugzilla.mozilla.org/form.web.bounty diff --git a/README.rst b/README.rst deleted file mode 100644 index bafc75445fec..000000000000 --- a/README.rst +++ /dev/null @@ -1,35 +0,0 @@ -.. image:: https://img.shields.io/badge/%E2%9D%A4-code%20of%20conduct-blue.svg - :target: https://github.com/mozilla/addons-server/blob/master/.github/CODE_OF_CONDUCT.md - :alt: Code of conduct - -.. image:: https://circleci.com/gh/mozilla/addons-server.svg?style=svg - :target: https://circleci.com/gh/mozilla/addons-server - - -Addons-Server -============= - -Welcome to the Addons Server repository! Please feel free to visit the web page of the current project hosted on `addons.mozilla.org`_. If you want to install it follow our guide located in `install docs`_. We'd love your help! You can come talk to us on `Matrix #amo:mozilla.org`_ if you have any questions. - -Please report bugs here: https://github.com/mozilla/addons/issues or https://github.com/mozilla/addons-server/issues -You can access the AMO dev environment at https://addons-dev.allizom.org/ and the AMO stage environment at https://addons.allizom.org/ - - -.. _`addons.mozilla.org`: https://addons.mozilla.org -.. _`install docs`: https://addons-server.readthedocs.io/en/latest/topics/install/docker.html -.. _`Matrix #amo:mozilla.org`: https://chat.mozilla.org/#/room/#amo:mozilla.org - - -.. marker-for-security-bug-inclusion-do-not-remove - -Security Bug Reports --------------------- - -This code and its associated production web page are included in the Mozilla’s web and services `bug bounty program`_. If you find a security vulnerability, please submit it via the process outlined in the program and `FAQ pages`_. Further technical details about this application are available from the `Bug Bounty Onramp page`_. - -Please submit all security-related bugs through Bugzilla using the `web security bug form`_. Never submit security-related bugs through a Github Issue or by email. - -.. _bug bounty program: https://www.mozilla.org/en-US/security/web-bug-bounty/ -.. _FAQ pages: https://www.mozilla.org/en-US/security/bug-bounty/faq-webapp/ -.. _Bug Bounty Onramp page: https://wiki.mozilla.org/Security/BugBountyOnramp/ -.. _web security bug form: https://bugzilla.mozilla.org/form.web.bounty diff --git a/babel.cfg b/babel.cfg index 18880252cddb..8a10f0e960a9 100644 --- a/babel.cfg +++ b/babel.cfg @@ -4,3 +4,6 @@ [django: src/olympia/**/templates/admin/**.html] [django: src/olympia/**/templates/devhub/forms/widgets/compat_app_input_option.html] [jinja2_custom: src/olympia/**/templates/**.html] + +[extractors] +jinja2_custom = olympia.core.babel:extract_jinja diff --git a/conftest.py b/conftest.py index eb8ccba583a7..d8a9e55aa6fd 100644 --- a/conftest.py +++ b/conftest.py @@ -4,6 +4,7 @@ Please note that there should not be any Django/Olympia related imports on module-level, they should instead be added to hooks or fixtures directly. """ + import os import uuid @@ -138,11 +139,12 @@ def test_pre_setup(request, tmpdir, settings): from django.core.cache import caches from django.utils import translation - from olympia import amo, core - from olympia.translations.hold import clean_translations from waffle import models as waffle_models from waffle.utils import get_cache as waffle_get_cache + from olympia import amo, core + from olympia.translations.hold import clean_translations + # Clear all cache-instances. They'll be re-initialized by Django # This will make sure that our random `KEY_PREFIX` is applied # appropriately. @@ -184,8 +186,6 @@ def _path(*args): settings.SHARED_STORAGE = shared_storage = _path(storage_root, 'shared_storage') settings.ADDONS_PATH = _path(storage_root, 'files') - settings.GUARDED_ADDONS_PATH = _path(storage_root, 'guarded-addons') - settings.GIT_FILE_STORAGE_PATH = _path(storage_root, 'git-storage') settings.MLBF_STORAGE_PATH = _path(storage_root, 'mlbf') settings.MEDIA_ROOT = _path(shared_storage, 'uploads') settings.SITEMAP_STORAGE_PATH = _path(storage_root, 'sitemaps') diff --git a/deps/.keep b/deps/.keep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 000000000000..c1944889b061 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,30 @@ +group "default" { + targets = ["web"] +} + +variable DOCKER_BUILD {} +variable DOCKER_COMMIT {} +variable DOCKER_VERSION {} +variable DOCKER_TARGET {} +variable DOCKER_TAG {} + +target "web" { + context = "." + dockerfile = "Dockerfile" + target = "${DOCKER_TARGET}" + tags = ["${DOCKER_TAG}"] + platforms = ["linux/amd64"] + args = { + DOCKER_COMMIT = "${DOCKER_COMMIT}" + DOCKER_VERSION = "${DOCKER_VERSION}" + DOCKER_BUILD = "${DOCKER_BUILD}" + DOCKER_TARGET = "${DOCKER_TARGET}" + DOCKER_SOURCE = "https://github.com/mozilla/addons-server" + } + pull = true + + output = [ + "type=docker", + ] + +} diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 0a226055e9ab..13bbe9968adc 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,5 +1,3 @@ -version: "2.4" - services: nginx: ports: diff --git a/docker-compose.private.yml b/docker-compose.private.yml index a0dbc2ebd0b9..6f6f3587613f 100644 --- a/docker-compose.private.yml +++ b/docker-compose.private.yml @@ -1,9 +1,8 @@ -version: "2.4" - services: worker: depends_on: - customs + - source-builder customs: build: @@ -15,3 +14,20 @@ services: - "nginx:olympia.test" depends_on: - web + + source-builder: + build: + context: ./private/addons-builder + environment: + - PORT=10200 + - API_AUTH_TOKEN=sourcebuildersecret + expose: + - 10200 + links: + - "nginx:olympia.test" + depends_on: + - web + networks: + default: + aliases: + - source-builder.test diff --git a/docker-compose.tools.yml b/docker-compose.tools.yml new file mode 100644 index 000000000000..a4606424f359 --- /dev/null +++ b/docker-compose.tools.yml @@ -0,0 +1,18 @@ +name: test_actions + +services: + base: + image: scratch + working_dir: /workdir + volumes: + - .:/workdir + + actionlint: + extends: base + image: rhysd/actionlint:1.7.9@sha256:a0383f60d92601e2694e24b24d37df7b6a40bed7cedbc447611c50009bf02d94 + + zizmor: + extends: base + image: ghcr.io/zizmorcore/zizmor:1.18.0@sha256:c5bbdb28b75702f181695d7a878e562ccb5c0a01847db87edda7476908d73dd6 + environment: + - GH_TOKEN diff --git a/docker-compose.yml b/docker-compose.yml index 85a63cfddb65..4a7b99d2947b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ -version: "2.4" - x-env-mapping: &env + # https://docs.docker.com/compose/environment-variables/envvars-precedence/ + env_file: + - .env environment: - CELERY_BROKER_URL=amqp://olympia:olympia@rabbitmq/olympia - CELERY_RESULT_BACKEND=redis://redis:6379/1 @@ -9,64 +10,98 @@ x-env-mapping: &env - MEMCACHE_LOCATION=memcached:11211 - MYSQL_DATABASE=olympia - MYSQL_ROOT_PASSWORD=docker - - OLYMPIA_SITE_URL=http://olympia.test + - SITE_URL=http://olympia.test - PYTHONDONTWRITEBYTECODE=1 - PYTHONUNBUFFERED=1 + - PYTHONBREAKPOINT=ipdb.set_trace - TERM=xterm-256color - - CIRCLECI=${CIRCLECI} - HISTFILE=/data/olympia/docker/artifacts/bash_history - HISTSIZE=50000 - HISTIGNORE=ls:exit:"cd .." - HISTCONTROL=erasedups - # Note: docker compose uses the values exported from .env for GID/UID if - # they exist. ./docker/fix_olympia_user.sh uses those variables to fix - # the uid/gid of the user to match the host if necessary. - - UID=${UID:-9500} - - GID=${UID:-9500} + - ENV=local + - SWAGGER_UI_ENABLED=true + - CIRCLECI + - SKIP_DATA_SEED + - FXA_CLIENT_ID + - FXA_CLIENT_SECRET + - STATIC_URL_PREFIX + - MEDIA_URL_PREFIX + +x-olympia: &olympia + <<: *env + image: ${DOCKER_TAG:-} + # We don't want docker compose to manage the image for us. + # We sometimes build the image locally and sometimes pull from a registry + # but docker compose should always assume the image is available. + pull_policy: never + # We drop down to a different user through entrypoint.sh, but starting as + # root allows us to fix the ownership of files generated at image build + # time through the ./docker/entrypoint.sh script. + user: root + platform: linux/amd64 + entrypoint: ["/data/olympia/docker/entrypoint.sh"] services: - worker: &worker - <<: *env - image: mozilla/addons-server:latest - # We drop down to a different user through supervisord, but starting as - # root allows us to fix the ownership of files generated at image build - # time through the ./docker/fix_olympia_user.sh script. - user: root - platform: linux/amd64 - command: - - /bin/sh - - -c - - | - ./docker/fix_olympia_user.sh - supervisord -n -c /data/olympia/docker/supervisor-celery.conf + static: + <<: *olympia + ports: + - "5173:5173" + volumes: + - ./:/data/olympia + - ./deps:/data/olympia/deps + command: make run_vite + + worker: + <<: *olympia + command: [ + "DJANGO_SETTINGS_MODULE=settings", + "watchmedo", + "auto-restart", + "--directory=/data/olympia/src", + "--pattern=*.py", + "--recursive", + "--no-restart-on-command-exit", + "--", + "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO", + ] volumes: - - .:/data/olympia - - ./deps:/deps - - ./package.json:/deps/package.json - - ./package-lock.json:/deps/package-lock.json + - ./:/data/olympia extra_hosts: - "olympia.test:127.0.0.1" + restart: on-failure:5 + healthcheck: + test: ["CMD-SHELL", "./manage.py monitors --services celery_worker --skip-checks"] + interval: 30s + retries: 3 + start_interval: 1s + start_period: 10s + depends_on: + - mysqld + - elasticsearch + - redis + - memcached + - rabbitmq + - autograph web: - <<: *worker - platform: linux/amd64 + extends: + service: worker command: - - /bin/sh - - -c - - | - ./docker/fix_olympia_user.sh - supervisord -n -c /data/olympia/docker/supervisor.conf + - DJANGO_SETTINGS_MODULE=settings uwsgi --ini /data/olympia/docker/uwsgi.ini + healthcheck: + test: ["CMD-SHELL", "./manage.py monitors --services localdev_web --skip-checks"] + interval: 30s + retries: 3 + start_interval: 1s + start_period: 10s nginx: - image: nginx + image: nginx:1.29-bookworm@sha256:8adbdcb969e2676478ee2c7ad333956f0c8e0e4c5a7463f4611d7a2e7a7ff5dc + <<: *env volumes: - - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf - - ./static:/srv/static - - ./site-static:/srv/site-static - - ./storage/shared_storage/uploads:/srv/user-media - - ./storage/files:/srv/user-media/addons - - ./storage/guarded-addons:/srv/user-media/guarded-addons - - ./storage/sitemaps:/srv/user-media/sitemaps + - data_nginx:/etc/nginx/templates + - ./:/srv ports: - "80:80" networks: @@ -75,30 +110,44 @@ services: - olympia.test depends_on: - web - - addons-frontend memcached: - image: memcached:1.4 - # Remove this once we upgrade to a version that provides multi-platform images - platform: linux/amd64 + image: memcached:1.5.16@sha256:9655004969fb55e163c665f1c485f48b48784d3eb54c210882103f54d7d842d1 mysqld: - image: mysql:8.0 + image: mysql:8.0@sha256:0275a35e79c60caae68fac520602d9f6897feb9b0941a1471196b1a01760e581 environment: - MYSQL_ALLOW_EMPTY_PASSWORD=yes - MYSQL_DATABASE=olympia ports: - "3306:3306" + volumes: + - data_mysqld:/var/lib/mysql + command: + # Optimize for development speed over durability + - --innodb-flush-log-at-trx-commit=0 + - --innodb-buffer-pool-size=64M + - --innodb-log-buffer-size=8M + - --innodb-log-file-size=32M + # Skip DNS lookups + - --skip-name-resolve + # Disable performance schema for faster startup + - --performance-schema=OFF + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "--silent"] + start_interval: 1s + timeout: 2s + start_period: 10s + retries: 3 elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3 + image: docker.elastic.co/elasticsearch/elasticsearch:8.19.8@sha256:1b6a877f18352510860ee065f01472bd37d33ac5eb1d943e0b9ed366b149638c environment: # Disable all xpack related features to avoid unrelated logging # in docker logs. https://github.com/mozilla/addons-server/issues/8887 # This also avoids us to require authentication for local development # which simplifies the setup. - xpack.security.enabled=false - - xpack.monitoring.enabled=false - xpack.graph.enabled=false - xpack.watcher.enabled=false - "discovery.type=single-node" @@ -106,10 +155,12 @@ services: mem_limit: 2g redis: - image: redis:6.2 + image: redis:6.2-bookworm@sha256:c628c49a6ce4a68bfbd548d7957d3d52b2267a4cd4b4a16779a1ca0cd1126527 + volumes: + - data_redis:/data rabbitmq: - image: rabbitmq:3.8 + image: rabbitmq:3.13@sha256:4cf3d0116716920b9a9e72b47d8c5a562296a79e0b0d48df56eee51967d123a7 hostname: olympia expose: - "5672" @@ -117,17 +168,18 @@ services: - RABBITMQ_DEFAULT_USER=olympia - RABBITMQ_DEFAULT_PASS=olympia - RABBITMQ_DEFAULT_VHOST=olympia + volumes: + - data_rabbitmq:/var/lib/rabbitmq autograph: - image: mozilla/autograph:3.3.2 - platform: linux/amd64 - command: /go/bin/autograph -c /data/olympia/scripts/autograph_localdev_config.yaml + image: mozilla/autograph:7.5.5@sha256:8d448fab80666c589eb0ff4f1b1aac358222a12461630ef760a5fd4db66c63be + command: /go/bin/autograph -c /data/autograph/autograph_localdev_config.yaml volumes: - - .:/data/olympia + - data_autograph:/data/autograph addons-frontend: <<: *env - image: mozilla/addons-frontend:latest + image: mozilla/addons-frontend:2025.11.27@sha256:9746ca74acc49bf85000911f26b4317028195c81ea0800c439a37a374d4a9479 platform: linux/amd64 environment: # We change the proxy port (which is the main entrypoint) as well as the @@ -141,3 +193,32 @@ services: # exposed using webpack and not by the node app server). - 7011:7011 command: yarn amo:olympia + +networks: + default: + driver: bridge + enable_ipv6: false + +volumes: + # Volume for rabbitmq/redis to avoid anonymous volumes + data_rabbitmq: + data_redis: + data_mysqld: + # Keep this value in sync with Makefile-os + # External volumes must be manually created/destroyed + name: addons-server_data_mysqld + external: true + # Volume for nginx configuration + data_nginx: + driver: local + driver_opts: + type: none + o: bind + device: ${PWD}/docker/nginx + # Volume for autograph configuration + data_autograph: + driver: local + driver_opts: + type: none + o: bind + device: ${PWD}/docker/autograph diff --git a/docker/artifacts/uwsgi-reload-monitor b/docker/artifacts/uwsgi-reload-monitor deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/scripts/autograph_localdev_config.yaml b/docker/autograph/autograph_localdev_config.yaml similarity index 99% rename from scripts/autograph_localdev_config.yaml rename to docker/autograph/autograph_localdev_config.yaml index 4e53ac6f0928..8c27e71c4cd8 100644 --- a/scripts/autograph_localdev_config.yaml +++ b/docker/autograph/autograph_localdev_config.yaml @@ -1,8 +1,5 @@ # Note (1): Most of the configuration here got copied from # https://github.com/mozilla-services/autograph/blob/master/autograph.yaml -# -# Note (2): the content of the file is also embedded in `.circleci/config.yml`. -# Any change here should likely be duplicated. server: # This port should be perfectly free, the upstream default of 8000 @@ -135,7 +132,6 @@ signers: states: recommended: true recommended-android: true - verified: true line: true relative_start: 0h duration: 26298h @@ -255,3 +251,7 @@ authorizations: key: 9vh6bhlc10y63ow2k4zke7k0c3l9hpr8mo96p92jmbfqngs9e7d signers: - webextensions-rsa-with-recommendation + +heartbeat: + hsmchecktimeout: 100ms + dbchecktimeout: 150ms diff --git a/docker/debian_packages.txt b/docker/debian_packages.txt new file mode 100644 index 000000000000..9d7df389bd9c --- /dev/null +++ b/docker/debian_packages.txt @@ -0,0 +1,27 @@ +# General (dev-) dependencies +bash-completion +build-essential +curl +libjpeg-dev +libsasl2-dev +libxml2-dev +locales +zlib1g-dev +libffi-dev +libssl-dev +nodejs +# Git, because we're using git-checkout dependencies +git +# Dependencies for mysql-python (from mysql apt repo, not debian) +pkg-config +mysql-client +libmysqlclient-dev +swig +gettext +# Use rsvg-convert to render our static theme previews +librsvg2-bin +# Use pngcrush to optimize the PNGs uploaded by developers +pngcrush +# pygraphviz is required to graph our ORM models +graphviz +libgraphviz-dev diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 000000000000..06aca156b3c0 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +### This is the entrypoint script used for local and CI environments +### It allows the web/worker containers to be run as root, but execute +### the commands as the olympia user. This is necessary because the +### id of the olympia user sometimes should match the host user's id +### to avoid permission issues with mounted volumes. + +set -ue + +if [[ $(id -u) -ne 0 ]]; then + echo "This script must be run as root" + exit 1 +fi + +OLYMPIA_USER="olympia" + +function get_olympia_uid() { echo "$(id -u "$OLYMPIA_USER")"; } +function get_olympia_gid() { echo "$(id -g "$OLYMPIA_USER")"; } + +OLD_HOST_UID=$(get_olympia_uid) + +# If the olympia user's uid is different in the container than from the build, +# we need to update the olympia user's uid to match the new one. +if [[ "${HOST_UID}" != "${OLD_HOST_UID}" ]]; then + usermod -u ${HOST_UID} ${OLYMPIA_USER} + echo "${OLYMPIA_USER} UID: ${OLD_HOST_UID} -> ${HOST_UID}" +fi + +NEW_HOST_UID=$(get_olympia_uid) +OLYMPIA_ID_STRING="${NEW_HOST_UID}:$(get_olympia_gid)" + +cat <, and +covers development using Add-ons Server, the source code for [Add-ons](https://addons.mozilla.org/). + +Its source location is in the [/docs](https://github.com/mozilla/addons-server/tree/master/docs) folder. + +Note: this project was once called *olympia*, this documentation often uses +that term. + +## Build the documentation + +This is as simple as running: + +``` +make docs +``` + +This is the same as `cd`'ing to the `docs` folder, and running `make +html` from there. + +We include a daemon that can watch and regenerate the built HTML when +documentation source files change. To use it, go to the `docs` folder +and run: + +``` +python watcher.py 'make html' $(find . -name '*.rst') +``` + +Once done, check the result by opening the following file in your browser: + +> /path/to/olympia/docs/\_build/html/index.html diff --git a/docs/README.rst b/docs/README.rst deleted file mode 100644 index c7138c30ab9d..000000000000 --- a/docs/README.rst +++ /dev/null @@ -1,47 +0,0 @@ -============================ -Add-ons Server Documentation -============================ - -This is the documentation for the use of the addons-server and its services. -All documentation is in plain text files using -`reStructuredText `_ and -`Sphinx `_. - -To build the documentation, you need the dependencies from -``requirements/docs.txt``. Those are automatically installed together with -``make update_deps``, so if you've installed that already (following the -:ref:`installation` page), you're all set. - -If you're unsure, activate your ``virtualenv`` and run:: - - make update_deps - -The documentation is viewable at http://addons-server.readthedocs.io/, and -covers development using Add-ons Server, the source code for `Add-ons -`_. - -Its source location is in the `/docs -`_ folder. - -Note: this project was once called *olympia*, this documentation often uses -that term. - -Build the documentation ------------------------ - -This is as simple as running:: - - make docs - -This is the same as ``cd``'ing to the ``docs`` folder, and running ``make -html`` from there. - -We include a daemon that can watch and regenerate the built HTML when -documentation source files change. To use it, go to the ``docs`` folder -and run:: - - python watcher.py 'make html' $(find . -name '*.rst') - -Once done, check the result by opening the following file in your browser: - - /path/to/olympia/docs/_build/html/index.html diff --git a/docs/_static/images/migration_graph_mermaid.png b/docs/_static/images/migration_graph_mermaid.png new file mode 100644 index 000000000000..c1c6a7f785e5 Binary files /dev/null and b/docs/_static/images/migration_graph_mermaid.png differ diff --git a/docs/_static/images/statistics-overview-chart.png b/docs/_static/images/statistics-overview-chart.png new file mode 100644 index 000000000000..d840118a83f7 Binary files /dev/null and b/docs/_static/images/statistics-overview-chart.png differ diff --git a/docs/_static/panzoom-graphviz.js b/docs/_static/panzoom-graphviz.js new file mode 100644 index 000000000000..35a35a160273 --- /dev/null +++ b/docs/_static/panzoom-graphviz.js @@ -0,0 +1,62 @@ +// Enable pan/zoom on all Graphviz SVGs after the DOM is loaded +// Requires panzoom.min.js to be loaded first + +document.addEventListener('DOMContentLoaded', () => { + const params = new URLSearchParams(window.location.search); + let active = params.has('zoom'); + let graph = document.querySelector('.panzoom-graph'); + let container = graph?.parentElement; + + if (!window.panzoom || !graph || !container) return; + + function toggleZoom(active) { + if (active) { + params.set('zoom', ''); + } else { + params.delete('zoom'); + } + window.location.search = params.toString(); + } + + if (!active) { + return graph.addEventListener('click', (e) => { + e.preventDefault(); + toggleZoom(true); + }); + } + + function initToggleButton() { + let btn = document.createElement('button'); + btn.id = 'panzoom-toggle'; + btn.classList.add('btn', 'btn-neutral'); + btn.textContent = 'Go Back'; + container.appendChild(btn); + return btn; + } + + let toggleButton = initToggleButton(); + + container.style.position = 'fixed'; + container.style.top = '0'; + container.style.left = '0'; + container.style.width = '100vw'; + container.style.height = '100vh'; + container.style.backgroundColor = 'white'; + container.style.zIndex = '1000'; + + toggleButton.style.position = 'fixed'; + toggleButton.style.top = '10px'; + toggleButton.style.right = '10px'; + + window.panzoom(graph, { + initialX: 0, + initialY: 0, + initialZoom: 1, + minZoom: 1, + transformOrigin: {x: 0, y: 0}, + }); + + toggleButton.addEventListener('click', () => { + toggleZoom(false); + }); +}); diff --git a/docs/conf.py b/docs/conf.py index 7e1b03209d6c..aa7265c9f7e1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,8 +15,6 @@ import sys import os -import sphinx_rtd_theme - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -32,6 +30,7 @@ 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', + 'sphinx.ext.graphviz', 'extensions.src_role', 'sphinxcontrib.httpdomain', 'sphinxcontrib.jquery', @@ -51,8 +50,8 @@ master_doc = 'index' # General information about the project. -project = u'addons-server' -copyright = u'2023, Mozilla' +project = 'addons-server' +copyright = '2024, Mozilla' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -114,7 +113,6 @@ on_rtd = os.environ.get('READTHEDOCS', None) == 'True' html_theme = 'sphinx_rtd_theme' -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # otherwise, readthedocs.org uses their theme by default, so no need to specify # it @@ -146,7 +144,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ['_static', '../node_modules/panzoom/dist'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -243,3 +241,12 @@ ('http:obj', 'array|null'), ('http:obj', 'int|null'), ] + +# Graphviz configuration +graphviz_output_format = 'png' + +# Add custom JS for pan/zoom on Graphviz diagrams +html_js_files = [ + 'panzoom.min.js', + 'panzoom-graphviz.js', +] diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000000..de5ab111e4af --- /dev/null +++ b/docs/index.md @@ -0,0 +1,43 @@ +# Welcome to Add-ons Servers documentation! + +Add-ons Server is the codebase for ; +the source lives at . + +In the past, this project was *olympia*; documentation that refers to olympia +refers to this project. + +## Contents + +```{toctree} +:maxdepth: 2 + +topics/readme_include +topics/api/index +topics/development/index +topics/models/index +topics/logs +topics/remote_addr +topics/third-party +topics/blocklist +topics/environments +topics/runbooks/index +``` + +## Archived Contents + +```{toctree} +:maxdepth: 1 +:titlesonly: true + +topics/api/v3_legacy/index +topics/api/v4_frozen/index +``` + +## Playground + +```{toctree} +:maxdepth: 1 +:titlesonly: true + +topics/playground +``` diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 2cfb924c5e7f..000000000000 --- a/docs/index.rst +++ /dev/null @@ -1,32 +0,0 @@ -========================================= -Welcome to Add-ons Servers documentation! -========================================= - -Add-ons Server is the codebase for https://addons.mozilla.org/; -the source lives at https://github.com/mozilla/addons-server. - -In the past, this project was *olympia*; documentation that refers to olympia -refers to this project. - -Contents --------- - -.. toctree:: - :maxdepth: 2 - - topics/readme_include - topics/api/index - topics/install/index - topics/development/index - topics/third-party - topics/blocklist - -Archived Contents ------------------ - -.. toctree:: - :titlesonly: - :maxdepth: 1 - - topics/api/v3_legacy/index - topics/api/v4_frozen/index diff --git a/docs/topics/api/abuse.rst b/docs/topics/api/abuse.rst index aa9e5f1144dd..ea29003506d0 100644 --- a/docs/topics/api/abuse.rst +++ b/docs/topics/api/abuse.rst @@ -56,6 +56,8 @@ to if necessary. :`. :`. + :`. :>json object|null reporter: The user who submitted the report, if authenticated. :>json int reporter.id: The id of the user who submitted the report. :>json string reporter.name: The name of the user who submitted the report. @@ -86,6 +88,8 @@ to if necessary. :>json string|null operating_system: The client's operating system. :>json string|null operating_system_version: The client's operating system version. :>json string|null reason: The reason for the report. + :>json string|null illegal_category: The type of illegal content - only defined when the reason is set to ``illegal``. + :>json string|null illegal_subcategory: The specific violation - only defined when the reason is set to ``illegal``. .. _abuse-report_entry_point-parameter: @@ -228,6 +232,102 @@ to if necessary. both Offending content is in both locations =========================== =================================================== +.. _abuse-report-illegal_category-parameter: + + Accepted values for the ``illegal_category`` parameter: + + ================================================ ================================================ + Value Description + ================================================ ================================================ + animal_welfare Animal welfare + consumer_information Consumer information infringements + data_protection_and_privacy_violations Data protection and privacy violations + illegal_or_harmful_speech Illegal or harmful speech + intellectual_property_infringements Intellectual property infringements + negative_effects_on_civic_discourse_or_elections Negative effects on civic discourse or elections + non_consensual_behaviour Non-consensual behavior + pornography_or_sexualized_content Pornography or sexualized content + protection_of_minors Protection of minors + risk_for_public_security Risk for public security + scams_and_fraud Scams or fraud + self_harm Self-harm + unsafe_and_prohibited_products Unsafe, non-compliant, or prohibited products + violence Violence + other Other + ================================================ ================================================ + +.. _abuse-report-illegal_subcategory-parameter: + + Accepted values for the ``illegal_subcategory`` parameter: + + ================================================ ============================================ ============================================================================================= + Illegal category Value Description + ================================================ ============================================ ============================================================================================= + animal_welfare other Something else + consumer_information insufficient_information_on_traders Insufficient information on traders + consumer_information noncompliance_pricing Non-compliance with pricing regulations + consumer_information hidden_advertisement Hidden advertisement or commercial communication, including by influencers + consumer_information misleading_info_goods_services Misleading information about the characteristics of the goods and services + consumer_information misleading_info_consumer_rights Misleading information about the consumer’s rights + consumer_information other Something else + data_protection_and_privacy_violations biometric_data_breach Biometric data breach + data_protection_and_privacy_violations missing_processing_ground Missing processing ground for data + data_protection_and_privacy_violations right_to_be_forgotten Right to be forgotten + data_protection_and_privacy_violations data_falsification Data falsification + data_protection_and_privacy_violations other Something else + illegal_or_harmful_speech defamation Defamation + illegal_or_harmful_speech discrimination Discrimination + illegal_or_harmful_speech hate_speech Illegal incitement to violence and hatred based on protected characteristics (hate speech) + illegal_or_harmful_speech other Something else + intellectual_property_infringements design_infringement Design infringements + intellectual_property_infringements geographic_indications_infringement Geographical indications infringements + intellectual_property_infringements patent_infringement Patent infringements + intellectual_property_infringements trade_secret_infringement Trade secret infringements + intellectual_property_infringements other Something else + negative_effects_on_civic_discourse_or_elections violation_eu_law Violation of EU law relevant to civic discourse or elections + negative_effects_on_civic_discourse_or_elections violation_national_law Violation of national law relevant to civic discourse or elections + negative_effects_on_civic_discourse_or_elections misinformation_disinformation_disinformation Misinformation, disinformation, foreign information manipulation and interference + negative_effects_on_civic_discourse_or_elections other Something else + non_consensual_behaviour non_consensual_image_sharing Non-consensual image sharing + non_consensual_behaviour non_consensual_items_deepfake Non-consensual items containing deepfake or similar technology using a third party's features + non_consensual_behaviour online_bullying_intimidation Online bullying/intimidation + non_consensual_behaviour stalking Stalking + non_consensual_behaviour other Something else + pornography_or_sexualized_content adult_sexual_material Adult sexual material + pornography_or_sexualized_content image_based_sexual_abuse Image-based sexual abuse (excluding content depicting minors) + pornography_or_sexualized_content other Something else + protection_of_minors age_specific_restrictions_minors Age-specific restrictions concerning minors + protection_of_minors child_sexual_abuse_material Child sexual abuse material + protection_of_minors grooming_sexual_enticement_minors Grooming/sexual enticement of minors + protection_of_minors other Something else + risk_for_public_security illegal_organizations Illegal organizations + risk_for_public_security risk_environmental_damage Risk for environmental damage + risk_for_public_security risk_public_health Risk for public health + risk_for_public_security terrorist_content Terrorist content + risk_for_public_security other Something else + scams_and_fraud inauthentic_accounts Inauthentic accounts + scams_and_fraud inauthentic_listings Inauthentic listings + scams_and_fraud inauthentic_user_reviews Inauthentic user reviews + scams_and_fraud impersonation_account_hijacking Impersonation or account hijacking + scams_and_fraud phishing Phishing + scams_and_fraud pyramid_schemes Pyramid schemes + scams_and_fraud other Something else + self_harm content_promoting_eating_disorders Content promoting eating disorders + self_harm self_mutilation Self-mutilation + self_harm suicide Suicide + self_harm other Something else + unsafe_and_prohibited_products prohibited_products Prohibited or restricted products + unsafe_and_prohibited_products unsafe_products Unsafe or non-compliant products + unsafe_and_prohibited_products other Something else + violence coordinated_harm Coordinated harm + violence gender_based_violence Gender-based violence + violence human_exploitation Human exploitation + violence human_trafficking Human trafficking + violence incitement_violence_hatred General calls or incitement to violence and/or hatred + violence other Something else + other other Something else + ================================================ ============================================ ============================================================================================= + ------------------------------ Submitting a user abuse report @@ -249,6 +349,8 @@ so reports can be responded to if necessary. :`. :`. + :`. :>json object|null reporter: The user who submitted the report, if authenticated. :>json int reporter.id: The id of the user who submitted the report. :>json string reporter.name: The name of the user who submitted the report. @@ -263,6 +365,8 @@ so reports can be responded to if necessary. :>json string user.username: The username of the user reported. :>json string message: The body/content of the abuse report. :>json string|null lang: The language code of the locale used by the client for the application. + :>json string|null illegal_category: The type of illegal content - only defined when the reason is set to ``illegal``. + :>json string|null illegal_subcategory: The specific violation - only defined when the reason is set to ``illegal``. .. _abuse-user-reason-parameter: @@ -298,6 +402,8 @@ so reports can be responded to if necessary. :`. :`. + :`. :>json object|null reporter: The user who submitted the report, if authenticated. :>json int reporter.id: The id of the user who submitted the report. :>json string reporter.name: The name of the user who submitted the report. @@ -310,6 +416,8 @@ so reports can be responded to if necessary. :>json string message: The body/content of the abuse report. :>json string|null lang: The language code of the locale used by the client for the application. :>json string|null reason: The reason for the report. + :>json string|null illegal_category: The type of illegal content - only defined when the reason is set to ``illegal``. + :>json string|null illegal_subcategory: The specific violation - only defined when the reason is set to ``illegal``. .. _abuse-rating-reason-parameter: @@ -345,6 +453,8 @@ so reports can be responded to if necessary. :`. :`. + :`. :>json object|null reporter: The user who submitted the report, if authenticated. :>json int reporter.id: The id of the user who submitted the report. :>json string reporter.name: The name of the user who submitted the report. @@ -356,6 +466,8 @@ so reports can be responded to if necessary. :>json int collection.id: The id of the collection reported. :>json string message: The body/content of the abuse report. :>json string|null lang: The language code of the locale used by the client for the application. + :>json string|null illegal_category: The type of illegal content - only defined when the reason is set to ``illegal``. + :>json string|null illegal_subcategory: The specific violation - only defined when the reason is set to ``illegal``. .. _abuse-collection-reason-parameter: diff --git a/docs/topics/api/activity.rst b/docs/topics/api/activity.rst index dd85ab8d0031..700e43fa99c5 100644 --- a/docs/topics/api/activity.rst +++ b/docs/topics/api/activity.rst @@ -55,6 +55,7 @@ This endpoint allows you to fetch a single review note for a specific version of :>json string user.name: The name of the reviewer or author. :>json string comments: The text content of the review note. :>json string date: The date the review note was created. + :>json string|null attachment_url: The link to download the associated attachment, if any .. _review-note-action: diff --git a/docs/topics/api/addons.rst b/docs/topics/api/addons.rst index 17f46e9d483c..a40347253078 100644 --- a/docs/topics/api/addons.rst +++ b/docs/topics/api/addons.rst @@ -25,6 +25,7 @@ This endpoint allows you to search through public add-ons. :query string author: Filter by exact (listed) author username or user id. Multiple author usernames or ids can be specified, separated by comma(s), in which case add-ons with at least one matching author are returned. :query string category: Filter by :ref:`category slug `. ``app`` and ``type`` parameters need to be set, otherwise this parameter is ignored. :query string color: (Experimental) Filter by color in RGB hex format, trying to find themes that approximately match the specified color. Only works for static themes. + :query string created: Filter to add-on that have a creation date matching a :ref:`threshold value `. ``YYYY-MM-DD``, ``YYYY-MM-DDTHH:MM``, ``YYYY-MM-DDTHH:MM:SS`` or milliseconds timestamps are supported. :query string exclude_addons: Exclude add-ons by ``slug`` or ``id``. Multiple add-ons can be specified, separated by comma(s). :query string guid: Filter by exact add-on guid. Multiple guids can be specified, separated by comma(s), in which case any add-ons matching any of the guids will be returned. As guids are unique there should be at most one add-on result per guid specified. For usage with Firefox, instead of separating multiple guids by comma(s), a single guid can be passed in base64url format, prefixed by the ``rta:`` string. :query string lang: Activate translations in the specific language for that query. (See :ref:`translated fields `) @@ -35,6 +36,7 @@ This endpoint allows you to search through public add-ons. :query string sort: The sort parameter. The available parameters are documented in the :ref:`table below `. :query string tag: Filter by exact tag name. Multiple tag names can be specified, separated by comma(s), in which case add-ons containing *all* specified tags are returned. See :ref:`available tags ` :query string type: Filter by :ref:`add-on type `. Multiple types can be specified, separated by comma(s), in which case add-ons that are any of the matching types are returned. + :query string updated: Filter to add-on that have a last updated date matching a :ref:`threshold value `. ``YYYY-MM-DD``, ``YYYY-MM-DDTHH:MM``, ``YYYY-MM-DDTHH:MM:SS`` or milliseconds timestamps are supported. :query string users: Filter to add-ons that have average daily users of a :ref:`threshold value `. :>json int count: The number of results for this query. :>json string next: The URL of the next page of results. @@ -79,7 +81,7 @@ This endpoint allows you to search through public add-ons. .. _addon-threshold-param: - Threshold style parameters allow queries against numeric values using comparison. + Threshold style parameters allow queries against numeric or date-time values using comparison. The following is supported (examples for query parameter `foo`): * greater than ``foo__gt`` (example query: ?foo__gt=10.1) @@ -162,7 +164,7 @@ This endpoint allows you to fetch a specific add-on by id, slug or guid. :>json string created: The date the add-on was created. :>json object current_version: Object holding the current :ref:`version ` of the add-on. For performance reasons the ``license`` field omits the ``text`` property from both the search and detail endpoints. :>json string default_locale: The add-on default locale for translations. - :>json object|null description: The add-on description (See :ref:`translated fields `). This field might contain some HTML tags. + :>json object|null description: The add-on description (See :ref:`translated fields `). This field might contain markdown. :>json object|null developer_comments: Additional information about the add-on provided by the developer. (See :ref:`translated fields `). :>json string edit_url: The URL to the developer edit page for the add-on. :>json string guid: The add-on `extension identifier `_. @@ -173,6 +175,7 @@ This endpoint allows you to fetch a specific add-on by id, slug or guid. :>json object icons: An object holding the URLs to an add-ons icon including a cachebusting query string as values and their size as properties. Currently exposes 32, 64, 128 pixels wide icons. :>json boolean is_disabled: Whether the add-on is disabled or not. :>json boolean is_experimental: Whether the add-on has been marked by the developer as experimental or not. + :>json boolean|null is_noindexed: Whether the add-on should be indexed or not indexed for SEO. Note that the search endpoint will always return a ``null`` value. :>json object|null name: The add-on name (See :ref:`translated fields `). :>json string last_updated: The date of the last time the add-on was updated by its developer(s). :>json object|null latest_unlisted_version: Object holding the latest unlisted :ref:`version ` of the add-on. This field is only present if the user has unlisted reviewer permissions, or is listed as a developer of the add-on. @@ -184,9 +187,9 @@ This endpoint allows you to fetch a specific add-on by id, slug or guid. :>json int position: The position in the list of previews images. :>json int previews[].thumbnail_size[]: width, height dimensions of of the preview image thumbnail. :>json string previews[].thumbnail_url: The URL (including a cachebusting query string) to the preview image thumbnail. - :>json object|null promoted: Object holding promotion information about the add-on. Null if the add-on is not currently promoted. - :>json string promoted.category: The name of the :ref:`promoted category ` for the add-on. - :>json array promoted.apps[]: Array of the :ref:`applications ` for which the add-on is promoted. + :>json array promoted: Array holding promotion information about the add-on. + :>json string promoted[].category: The name of the :ref:`promoted category ` for the add-on. + :>json array promoted[].apps[]: Array of the :ref:`applications ` for which the add-on is promoted. :>json object ratings: Object holding ratings summary information about the add-on. :>json int ratings.count: The total number of user ratings for the add-on. :>json int ratings.text_count: The number of user ratings with review text for the add-on. @@ -203,7 +206,7 @@ This endpoint allows you to fetch a specific add-on by id, slug or guid. :>json string review_url: The URL to the reviewer review page for the add-on. :>json string slug: The add-on slug. :>json string status: The :ref:`add-on status `. - :>json object|null summary: The add-on summary (See :ref:`translated fields `). This field supports "linkification" and therefore might contain HTML hyperlinks. + :>json object|null summary: The add-on summary (See :ref:`translated fields `). :>json object|null support_email: The add-on support email (See :ref:`translated fields `). :>json object|null support_url: The add-on support URL (See :ref:`translated fields ` and :ref:`Outgoing Links `). :>json array tags: List containing the tag names set on the add-on. @@ -278,14 +281,12 @@ This endpoint allows you to fetch a specific add-on by id, slug or guid. line "By Firefox" category notable Notable category recommended Recommended category - sponsored Sponsored category spotlight Spotlight category strategic Strategic category - verified Verified category badged A meta category that's available for the ``promoted`` search filter that is all the categories we expect an API client to expose as "reviewed" by Mozilla. - Currently equal to ``line&recommended&sponsored&verified``. + Currently equal to ``line&recommended``. ============== ========================================================== @@ -313,7 +314,7 @@ is compatible with. :` the add-on belongs to for a given :ref:`add-on application `. :`_. :` and custom license text is fixed to `en-US`. - :`). This field can contain some HTML tags. + :`). This field can contain some Markdown. :`). :` and :ref:`Outgoing Links `). :` the add-on belongs to for a given :ref:`add-on application `. :`_. :` and custom license text is fixed to `en-US`. - :`). This field can contain some HTML tags. + :`). This field can contain some Markdown. :`). :` and :ref:`Outgoing Links `). :` to upload a new icon. @@ -418,6 +419,7 @@ Delete This endpoint allows an add-on to be deleted. Because add-on deletion is an irreversible and destructive action an additional token must be retrieved beforehand, and passed as a parameter to the delete endpoint. Deleting the add-on will permanently delete all versions and files submitted for this add-on, listed or not. +All versions will be soft-blocked (restricted), which will disable and prevent any further installation in Firefox. Existing users can choose to re-enable the add-on. The add-on ID (``guid``) cannot be restored and will forever be unusable for submission. .. note:: @@ -545,6 +547,8 @@ This endpoint allows you to fetch a single version belonging to a specific add-o :>json array file.optional_permissions[]: Array of the optional webextension permissions for this File, as strings. Empty for non-webextensions. :>json array file.host_permissions[]: Array of the host permissions for this File, as strings. Empty for non-webextensions. :>json array file.permissions[]: Array of the webextension permissions for this File, as strings. Empty for non-webextensions. + :>json array file.data_collection_permissions[]: Array of the data collection permissions for this File, as strings. Empty for non-webextensions. + :>json array file.optional_data_collection_permissions[]: Array of the optional data collection permissions for this File, as strings. Empty for non-webextensions. :>json int file.size: The size for the file, in bytes. :>json int file.status: The :ref:`status ` for the file. :>json string file.url: The (absolute) URL to download the file. @@ -736,7 +740,7 @@ Version Delete .. _version-delete: -This endpoint allows a version to be deleted. +This endpoint allows a version to be deleted. The version will be soft-blocked (restricted), which will disable and prevent any further installation in Firefox. Existing users can choose to re-enable the add-on. .. note:: This API requires :doc:`authentication `, and for the user to be an author of the add-on. @@ -747,6 +751,37 @@ This endpoint allows a version to be deleted. .. http:delete:: /api/v5/addons/addon/(int:addon_id|string:addon_slug|string:addon_guid)/versions/(int:id|string:version_number)/ +---------------- +Version Rollback +---------------- + +.. _version-rollback: + +This endpoint allows an already approved version to be immediately re-created with a new version number, to rollback a problematic version. + +The latest version in each channel is unavailable for rollback (or you wouldn't be rolling back anything); also, for listed channel, is limited to the previous approved version. +The rollback is asynchronous; you will be notified by email when it's complete (or failed). + +See :ref:`version create ` - the second version in the list - for the listed version available for rollback. Call the endpoint with ``?all_with_unlisted`` for all the unlisted versions, ignoring the first version and any versions that are not ``file.status == public``. + + .. note:: + This API requires :doc:`authentication `, and for the user to be an author of the add-on. + + .. note:: + This API accepts both version ids and version numbers in the URL. If the version number passed does not contain any dot characters (``.``) it would be considered an ``id``. To avoid this and force a lookup by version number, add a ``v`` prefix to it. + + .. warning:: + For the listed channel, if there is a version awaiting review in the listed channel it will be disabled as part of the rollback. + + +.. http:post:: /api/v5/addons/addon/(int:addon_id|string:addon_slug|string:addon_guid)/versions/(int:id|string:version_number)/rollback/ + + .. _version-rollback-request: + + :`). + + -------------- Preview Create -------------- @@ -894,6 +929,26 @@ This endpoint allows you to fetch an add-on EULA and privacy policy. :>json object|null privacy_policy: The text of the Privacy Policy, if present (See :ref:`translated fields `). +---------------------------- +EULA and Privacy Policy Edit +---------------------------- + +.. _addon-eula-policy-edit: + +This endpoint allows an add-on's EULA and privacy policy to be edited. + + .. note:: + This API requires :doc:`authentication `, and for the user to be an author of the add-on. + + .. note:: + This API is not valid for themes - themes do not have EULA or privacy policies. + +.. http:patch:: /api/v5/addons/addon/(int:id|string:slug|string:guid)/eula_policy/ + + :`). + :`). + + -------------- Language Tools -------------- @@ -939,7 +994,7 @@ Replacement Add-ons .. _addon-replacement-addons: -This endpoint returns a list of suggested replacements for legacy add-ons that are unsupported in Firefox 57. Added to support the TAAR recommendation service. +This endpoint returns a list of suggested replacements for legacy add-ons that are unsupported in Firefox 57. .. http:get:: /api/v5/addons/replacement-addon/ @@ -959,17 +1014,16 @@ Recommendations .. _addon-recommendations: -This endpoint provides recommendations of other addons to install, fetched from the `recommendation service `_. -Four recommendations are fetched, but only valid, publicly available addons are shown (so max 4 will be returned, and possibly less). +This endpoint provides recommendations of other addons to install. Maximum four recommendations will be returned. .. http:get:: /api/v5/addons/recommendations/ :query string app: Set the :ref:`add-on application ` for that query. This won't filter the results. Defaults to ``firefox``. :query string guid: Fetch recommendations for this add-on guid. :query string lang: Activate translations in the specific language for that query. (See :ref:`translated fields `) - :query boolean recommended: Fetch recommendations from the recommendation service, or return a curated fallback list instead. - :>json string outcome: Outcome of the response returned. Will be either: ``recommended`` - responses from recommendation service; ``recommended_fallback`` - service timed out or returned empty or invalid results so we returned fallback; ``curated`` - ``recommended=False`` was requested so fallback returned. - :>json string|null fallback_reason: if ``outcome`` was ``recommended_fallback`` then the reason why. Will be either: ``timeout``, ``no_results``, or ``invalid_results``. + :query boolean recommended: Ignored. + :>json string outcome: Outcome of the response returned. Will always be ``curated``. + :>json null fallback_reason: Always null. :>json int count: The number of results for this query. :>json string next: The URL of the next page of results. :>json string previous: The URL of the previous page of results. diff --git a/docs/topics/api/auth.rst b/docs/topics/api/auth.rst index 373d3ab2713a..cd58c607866e 100644 --- a/docs/topics/api/auth.rst +++ b/docs/topics/api/auth.rst @@ -29,12 +29,11 @@ To create JWTs, first obtain a **key** and **secret** from the If someone obtains your secret they can make API requests on behalf of your user account. -Create a JWT for each request -============================= +Create a JWT +============ -Prior to making every API request, you need to generate a fresh `JWT`_. -The JWT will have a short expiration time and is only valid for a single -request so you can't cache or reuse it. +Prior to making API requests, you need to generate a `JWT`_. +The JWT will have a short expiration time but can be re-used until they expire. You only need to include a few standard fields; here's what the raw JSON object needs to look like before it's signed: @@ -68,11 +67,6 @@ exp the *expiration time*. It should be a Unix epoch timestamp in UTC time and must be **no longer than five minutes** past the issued at time. - .. versionchanged:: 2016-10-06 - - We increased the expiration time from 60 seconds to five minutes - to workaround support for large and slow uploads. - .. note:: If you're having trouble authenticating, make sure your system diff --git a/docs/topics/api/blocklist.rst b/docs/topics/api/blocklist.rst index 7d9373643520..deb20ee359ba 100644 --- a/docs/topics/api/blocklist.rst +++ b/docs/topics/api/blocklist.rst @@ -26,9 +26,8 @@ This endpoint returns an add-on Block from the blocklist, specified by guid or i :>json string modified: The date the block was last updated. :>json object|null addon_name: The add-on name, if we have details of an add-on matching that guid (See :ref:`translated fields `). :>json string guid: The guid of the add-on being blocked. - :>json string min_version: The minimum version of the add-on that will be blocked. "0" is the lowest version, meaning all versions up to max_version will be blocked. ("0" - "*" would be all versions). - :>json string max_version: The maximum version of the add-on that will be blocked. "*" is the highest version, meaning all versions from min_version will be blocked. ("0" - "*" would be all versions). :>json string|null reason: Why the add-on needed to be blocked. :>json object|null url: A url to the report/request that detailed why the add-on should potentially be blocked. Typically a bug report on bugzilla.mozilla.org. (See :ref:`Outgoing Links `) - :>json string versions[]: The versions of this add-on that are blocked. + :>json string blocked[]: The versions of this add-on that are (hard) blocked. + :>json string soft_blocked[]: The versions of this add-on that are soft blocked (can be optionally re-enabled by existing users). :>json boolean is_all_versions: Are all versions of this add-on blocked. If ``False``, some versions are not blocked. diff --git a/docs/topics/api/discovery.rst b/docs/topics/api/discovery.rst index 3a9912cc4406..60a62e9e1247 100644 --- a/docs/topics/api/discovery.rst +++ b/docs/topics/api/discovery.rst @@ -21,25 +21,17 @@ Firefox (about:addons). .. note:: - If a telemetry client id is passed as a parameter to the discovery pane api - endpoint then static curated content is amended with recommendations from the - `recommendation service `_. The same number - of results will be returned as a standard discovery response and only extensions - (not themes) are recommended. Only valid, publicly available addons are shown. - - E.g. a standard discovery pane will display 7 items, 4 extensions and 3 themes. - Up to 4 extensions will be replaced with recommendations; the 3 themes will not - be replaced. The API will still return a total of 7 items. + A standard discovery pane will display 7 items, 4 extensions and 3 themes. .. http:get:: /api/v5/discovery/ - :query string lang: Activate translations in the specific language for that query. (See :ref:`translated fields `) + :query string lang: Activates translations. (See :ref:`translated fields `) :query string edition: Optionally return content for a specific edition of Firefox. Currently only ``china`` (and the alias ``MozillaOnline``) is supported. - :query string telemetry-client-id: Optional sha256 hash of the telemetry client ID to be passed to the TAAR service to enable recommendations. Must be the hex value of a sha256 hash, otherwise it will be ignored. + :query string telemetry-client-id: Ignored. Was previously an optional sha256 hash of the telemetry client ID to be passed to the TAAR service to enable recommendations. :>json int count: The number of results for this query. :>json array results: The array containing the results for this query. :>json object|null results[].description_text: The description for this item, if any. (See :ref:`translated fields `. Note: when ``lang`` is not specified, not all locales will be returned, unlike other translated fields). - :>json boolean results[].is_recommendation: If this item was from the recommendation service, rather than static curated content. + :>json boolean results[].is_recommendation: Always False. :>json object results[].addon: The :ref:`add-on ` for this item. Only a subset of fields are present: ``id``, ``authors``, ``average_daily_users``, ``current_version`` (with only the ``id``, ``compatibility``, ``is_strict_compatibility_enabled`` and ``file`` fields present), ``guid``, ``icon_url``, ``name``, ``ratings``, ``previews``, ``slug``, ``theme_data``, ``type`` and ``url``. @@ -51,8 +43,7 @@ Editorial Content This endpoint allows you to fetch all editorial content for Discovery Pane Recommendations. This is used internally to generate .po files containing the -strings defined by the content team. It is also used by TAAR service to obtain a list -of appropriate add-ons to recommended. +strings defined by the content team. .. http:get:: /api/v5/discovery/editorial/ diff --git a/docs/topics/api/licenses.rst b/docs/topics/api/licenses.rst index 36d1fb03e8fa..eae2de3b47ee 100644 --- a/docs/topics/api/licenses.rst +++ b/docs/topics/api/licenses.rst @@ -16,7 +16,7 @@ License Choices - non-Themes .. _license-list-extension: -Some popular opensource licenses can be chosen from, when creating or updating a version +Some popular open source licenses can be chosen from, when creating or updating a version via in the :ref:`add-on api`. These licenses cannot be used for themes. Open source license slugs are taken from the `SPDX License list `_ - and are case-sensitive. @@ -25,13 +25,17 @@ Open source license slugs are taken from the `SPDX License list `_ - GPL-2.0-or-later `GNU General Public License v2.0 (or later) `_ - GPL-3.0-or-later `GNU General Public License v3.0 (or later) `_ - LGPL-2.1-or-later `GNU Library General Public License v2.1 (or later) `_ - LGPL-3.0-or-later `GNU Library General Public License v3.0 (or later) `_ - MIT `The MIT License `_ - BSD-2-Clause `The BSD (2 clause) License `_ + MPL-2.0 `Mozilla Public License 2.0 `_ + Apache-2.0 `Apache License 2.0 `_ + GPL-2.0-only `GNU General Public License v2.0 only `_ + GPL-3.0-only `GNU General Public License v3.0 only `_ + LGPL-2.1-only `GNU Lesser General Public License v2.1 only `_ + LGPL-3.0-only `GNU Lesser General Public License v3.0 only `_ + AGPL-3.0-only `GNU Affero General Public License v3.0 only `_ + MIT `MIT License `_ + ISC `ISC License `_ + BSD-2-Clause `BSD 2-Clause "Simplified" License `_ + Unlicense `The Unlicense `_ =================== ============================================================== @@ -43,17 +47,17 @@ License Choices - Themes Creative Commons licenses can be chosen from, when creating or updating a theme version via in the :ref:`add-on api`. These are the only license options for themes. -Open source license slugs are taken from the `SPDX License list `_ +Creative Commons license slugs are taken from the `SPDX License list `_ - and are case-sensitive. ====================== =========================================================== Slug License ====================== =========================================================== cc-all-rights-reserved All Rights Reserved. This is not an open source license. - CC-BY-3.0 `Creative Commons Attribution 3.0 `_ - CC-BY-NC-3.0 `Creative Commons Attribution-NonCommercial 3.0 `_ - CC-BY-NC-ND-3.0 `Creative Commons Attribution-NonCommercial-NoDerivs 3.0 `_ - CC-BY-NC-SA-3.0 `Creative Commons Attribution-NonCommercial-Share Alike 3.0 `_ - CC-BY-ND-3.0 `Creative Commons Attribution-NoDerivs 3.0 `_ - CC-BY-SA-3.0 `Creative Commons Attribution-ShareAlike 3.0 `_ + CC-BY-4.0 `Creative Commons Attribution 4.0 `_ + CC-BY-NC-4.0 `Creative Commons Attribution-NonCommercial 4.0 `_ + CC-BY-NC-ND-4.0 `Creative Commons Attribution-NonCommercial-NoDerivs 4.0 `_ + CC-BY-NC-SA-4.0 `Creative Commons Attribution-NonCommercial-Share Alike 4.0 `_ + CC-BY-ND-4.0 `Creative Commons Attribution-NoDerivs 4.0 `_ + CC-BY-SA-4.0 `Creative Commons Attribution-ShareAlike 4.0 `_ ====================== =========================================================== diff --git a/docs/topics/api/overview.rst b/docs/topics/api/overview.rst index 1a9da2fd2ba5..1115644e5ba4 100644 --- a/docs/topics/api/overview.rst +++ b/docs/topics/api/overview.rst @@ -246,8 +246,9 @@ Note, if the field is also a translated field then the ``url`` and ``outgoing`` values could be an object rather than a string (See :ref:`translated fields ` for translated field representations). -Fields supporting some HTML, such as add-on ``description`` or ``summary``, -always wrap any links directly inside the content (the original url is not available). +Fields supporting some Markdown, such as add-on ``description`` or ``license``, +always wrap any links directly inside the content (the original url is not +available). ~~~~~~~~~~~~ @@ -397,6 +398,7 @@ v4 API changelog * 2021-01-14: as addons-frontend now uses /v5/, v5 becomes the stable default; v4 becomes frozen; v3 is deprecated * 2021-02-12: added ``versions_url`` to addon detail endpoint. https://github.com/mozilla/addons-server/issues/16534 * 2021-02-25: ``platform`` filtering was removed from add-on search and autocomplete endpoints. https://github.com/mozilla/addons-server/issues/16463 +* 2024-11-28: reviewers APIs to list, browse, compare and draft comments on versions were removed. ---------------- v5 API changelog @@ -469,6 +471,15 @@ These are `v5` specific changes - `v4` changes apply also. * 2023-11-02: removed ``application`` from categories endpoint, flattened ``categories`` in addon detail/search endpoint. https://github.com/mozilla/addons-server/issues/5989 * 2023-11-09: removed reviewers /enable and /disable endpoints. https://github.com/mozilla/addons-server/issues/21356 * 2023-12-07: added ``lang`` parameter to all /abuse/report/ endpoints. https://github.com/mozilla/addons-server/issues/21529 +* 2024-06-20: added ``illegal_category`` parameter to all /abuse/report/ endpoints. https://github.com/mozilla/addons/issues/14870 +* 2024-06-20: added ``illegal_subcategory`` parameter to all /abuse/report/ endpoints. https://github.com/mozilla/addons/issues/14875 +* 2024-08-08: added support for writing to add-on eula_policy endpoint. https://github.com/mozilla/addons/issues/14927 +* 2024-08-22: restricted add-on eula_policy endpoint to non-themes only. https://github.com/mozilla/addons/issues/14937 +* 2024-10-17: replaced ``versions`` with ``blocked`` and ``soft_blocked`` in blocklist api; dropped unused ``min_version`` and ``max_version``. https://github.com/mozilla/addons/issues/15015 +* 2025-06-26: added ``data_collection_permissions`` and ``optional_data_collection_permissions`` to the version detail endpoint (and therefore addons/search as well). https://github.com/mozilla/addons/issues/15620 +* 2025-09-01: added ``is_noindexed`` to the add-on detail/search endpoint. https://github.com/mozilla/addons/issues/15808 +* 2025-09-01: added ``created`` and ``updated`` query parameters to addon search api. https://github.com/mozilla/addons/issues/15814 +* 2025-09-18: added /rollback endpoint to version detail api. https://github.com/mozilla/addons/issues/15696 .. _`#11380`: https://github.com/mozilla/addons-server/issues/11380/ .. _`#11379`: https://github.com/mozilla/addons-server/issues/11379/ diff --git a/docs/topics/api/ratings.rst b/docs/topics/api/ratings.rst index a9ac3323e663..4dbaeec9451b 100644 --- a/docs/topics/api/ratings.rst +++ b/docs/topics/api/ratings.rst @@ -27,6 +27,7 @@ user has already posted a rating for the current version of an add-on. .. http:get:: /api/v5/ratings/rating/ :query string addon: The :ref:`add-on ` id, slug, or guid to fetch ratings from. When passed, the ratings shown will always be the latest posted by each user on this particular add-on (which means there should only be one rating per user in the results), unless the ``version`` parameter is also passed. + :query string app: Set the :ref:`add-on application ` for that query. This won't filter the results, but can affect the URLs being returned. Defaults to ``firefox``. :query string exclude_ratings: Exclude ratings by their ``id``. Multiple ratings can be specified, separated by comma(s). :query string filter: The :ref:`filter(s) ` to apply. :query string score: Only include ratings that have been given a specific ``score``. Multiple scores can be specified, separated by comma(s). diff --git a/docs/topics/api/reviewers.rst b/docs/topics/api/reviewers.rst index 20f2dec9549a..3018c797bdac 100644 --- a/docs/topics/api/reviewers.rst +++ b/docs/topics/api/reviewers.rst @@ -9,8 +9,11 @@ Reviewers These APIs are not frozen and can change at any time without warning. See :ref:`the API versions available` for alternatives if you need stability. - The only authentication method available at - the moment is :ref:`the internal one`. + + The only authentication method available for these APIs is + :ref:`the internal one`, except for the + :ref:`validation results` endpoint, which allows both + internal and :ref:`external auth`. --------- Subscribe @@ -50,6 +53,23 @@ sent when a new version is submitted on a particular add-on. .. http:post:: /api/v5/reviewers/addon/(int:addon_id)/unsubscribe_unlisted/ +--------------- +File Validation +--------------- + +.. _reviewers-validation: + +This endpoint allows you to view the validation results of a given file +belonging to an add-on. + + .. note:: + Requires authentication and the current user to have any + reviewer-related permission. + +.. http:post:: /api/v5/reviewers/addon/(int: addon_id)/file/(int: file_id)/validation/ + + :>json object validation: the validation results + ----- Flags ----- @@ -102,274 +122,3 @@ denied. :statuscode 202: Success. :statuscode 409: The add-on GUID was already denied. - -------------- -List Versions -------------- - -This endpoint allows you to list versions that can be used either for :ref:`browsing ` or diffing versions. - - .. note:: - Requires authentication and the current user to have ``ReviewerTools:View`` - permission for listed add-ons as well as ``Addons:ReviewUnlisted`` for - unlisted add-ons. Additionally the current user can also be the owner - of the add-on. - - This endpoint is not paginated as normal, and instead will return all - results without obeying regular pagination parameters. - - -If the user doesn't have ``AddonsReviewUnlisted`` permissions only listed versions are shown. Otherwise it can contain mixed listed and unlisted versions. - -.. http:get:: /api/v5/reviewers/addon/(int:addon_id)/versions/ - - :>json int id: The version id. - :>json string channel: The version channel, which determines its visibility on the site. Can be either ``unlisted`` or ``listed``. - :>json string version: The version number string for the version. - -.. _reviewers-versions-browse: - ------- -Browse ------- - -This endpoint allows you to browse through the contents of an Add-on version. - - .. note:: - Requires authentication and the current user to have ``ReviewerTools:View`` - permission for listed add-ons as well as ``Addons:ReviewUnlisted`` for - unlisted add-ons. Additionally the current user can also be the owner - of the add-on. - -.. http:get:: /api/v5/reviewers/addon/(int:addon_id)/versions/(int:version_id)/ - - Inherits the following properties from :ref:`version detail `: ``id``, ``channel``, ``reviewed`` and ``version``. - - .. _reviewers-versions-browse-detail: - - :param string file: The specific file in the XPI to retrieve. Defaults to manifest.json, install.rdf or package.json for Add-ons as well as the XML file for search engines. - :param boolean file_only: Indicates that the API should only return data for the requested file, and not version data. If this is ``true`` then the only property returned of those listed below is the ``file`` property. - :>json string validation_url_json: The absolute url to the addons-linter validation report, rendered as JSON. - :>json string validation_url: The absolute url to the addons-linter validation report, rendered as HTML. - :>json boolean has_been_validated: ``True`` if the version has been validated through addons-linter. - :>json object addon: A simplified :ref:`add-on ` object that contains only a few properties: ``id``, ``name``, ``icon_url`` and ``slug``. - :>json array file_entries[]: The complete file-tree of the extracted XPI. - :>json int file_entries[].depth: Level of folder-tree depth, starting with 0. - :>json string file_entries[].filename: The filename of the file. - :>json string file_entries[].path: The absolute path (from the root of the XPI) of the file. - :>json string file_entries[].mime_category: The mime type category of this file. Can be ``image``, ``directory``, ``text`` or ``binary``. - :>json object file: The requested file. - :>json int file.id: The id of the submitted file (i.e., the xpi file). - :>json string file.content: Raw content of the requested file. - :>json string file.selected_file: The selected file, either from the ``file`` parameter or the default (manifest.json, install.rdf or package.json for Add-ons as well as the XML file for search engines). - :>json string|null file.download_url: The download url of the selected file or ``null`` in case of a directory. - :>json string file.mimetype: The determined mimetype of the selected file or ``application/octet-stream`` if none could be determined. - :>json string file.sha256: SHA256 hash of the selected file. - :>json int file.size: The size of the selected file in bytes. - :>json string file.filename: The filename of the file. - :>json string file.mime_category: The mime type category of this file. Can be ``image``, ``directory``, ``text`` or ``binary``. - :>json boolean uses_unknown_minified_code: Indicates that the selected file could be using minified code. - - -------- -Compare -------- - -This endpoint allows you to compare two Add-on versions with each other. - - .. note:: - Requires authentication and the current user to have ``ReviewerTools:View`` - permission for listed add-ons as well as ``Addons:ReviewUnlisted`` for - unlisted add-ons. Additionally the current user can also be the owner - of the add-on. - -.. http:get:: /api/v5/reviewers/addon/(int:addon_id)/versions/(int:base_version_id)/compare_to/(int:version_id)/ - - .. note:: - - Contrary to what ``git diff`` does, this API renders a hunk full of unmodified lines for unmodified files. - - Inherits most properties from :ref:`browse detail `, except that most of the `file.entries[]` properties - and `file.download_url` can be `null` in case of a deleted file. - - Properties specific to this endpoint: - - :>json array file_entries[]: The complete file-tree of the extracted XPI. - :>json string file.entries[].status: Status of this file, see https://git-scm.com/docs/git-status#_short_format - :>json int file_entries[].depth: Level of folder-tree depth, starting with 0. - :>json string file_entries[].filename: The filename of the file. - :>json string file_entries[].path: The absolute path (from the root of the XPI) of the file. - :>json string file_entries[].mime_category: The mime type category of this file. Can be ``image``, ``directory``, ``text`` or ``binary``. - :>json object|null diff: See the following output with inline comments for a complete description. - :>json object base_file: The file attached to the base version you're comparing against. - :>json object base_file.id: The id of the base file. - :>json boolean uses_unknown_minified_code: Indicates that the selected file in either the current or the parent version could be using minified code. - - Git patch we're talking about: - - .. code:: diff - - diff --git a/README.md b/README.md - index a37979d..b12683c 100644 - --- a/README.md - +++ b/README.md - @@ -1 +1 @@ - -# beastify - +Updated readme - diff --git a/manifest.json b/manifest.json - index aba695f..24f385f 100644 - --- a/manifest.json - +++ b/manifest.json - @@ -1,36 +1 @@ - -{ - - - - "manifest_version": 2, - - "name": "Beastify", - - "version": "1.0", - - - - "permissions": [ - - "http://*/*", - - "https://*/*", - - "bookmarks", - - "made up permission", - - "https://google.com/" - - ], - - - - "content_scripts": [ - - { - - "matches": ["*://*.mozilla.org/*"], - - "js": ["borderify.js"] - - }, - - { - - "matches": ["*://*.mozilla.com/*", "https://*.mozillians.org/*"], - - "js": ["borderify.js"] - - } - - ], - - - - "browser_action": { - - "default_icon": "button/beasts.png", - - "default_title": "Beastify", - - "default_popup": "popup/choose_beast.html" - - }, - - - - "web_accessible_resources": [ - - "beasts/*.jpg" - - ] - - - -} - +{"id": "random"} - - - The following represents the git patch from above. - - .. code:: javascript - - "diff": { - "path": "README.md", - "old_path": "README.md", - "size": 15, // Size in bytes - "lines_added": 1, // How many lines got added - "lines_deleted": 1, // How many lines got deleted - "is_binary": false, // Is this a binary file (as determined by git) - "mode": "M", // Status of this file, see https://git-scm.com/docs/git-status#_short_format - "hunks": [ - { - "header": "@@ -1 +1 @@\\n", - "old_start": 1, - "new_start": 1, - "old_lines": 1, - "new_lines": 1, - "changes": [ - { - "content": "# beastify\\n", - "type": "delete", - "old_line_number": 1, - "new_line_number": -1 - }, - { - "content": "Updated readme\\n", - "type": "insert", - "old_line_number": -1, - "new_line_number": 1 - } - ] - } - ], - "parent": "075c5755198be472522477a1b396951b3b68ac18", - "hash": "00161dcf22afb7bab23cf205f0c903eb5aad5431" - } - - ------------------ -Drafting Comments ------------------ - -These endpoints allow you to draft comments that can be submitted through the regular reviewer pages. - - .. note:: - Requires authentication and the current user to have ``ReviewerTools:View`` - permission for listed add-ons as well as ``Addons:ReviewUnlisted`` for - unlisted add-ons. Additionally the current user can also be the owner - of the add-on. - - -.. http:get:: /api/v5/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/ - - Retrieve existing draft comments for a specific version. See :ref:`pagination ` for more details. - - :>json int count: The number of comments for this version. - :>json string next: The URL of the next page of results. - :>json string previous: The URL of the previous page of results. - :>json array results: An array of :ref:`comments `. - - -.. http:get:: /api/v5/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/(int:comment_id)/ - - .. _reviewers-draft-comment-detail-object: - - :>json int id: The id of the draft comment object. - :>json string comment: The comment that is being drafted as part of a review. Specific to a line in a file. - :>json string|null filename: The full file path a specific comment is related to. Can be ``null`` in case a comment doesn't belong to a specific file but the whole version. - :>json int|null lineno: The line number a specific comment is related to. Please make sure that in case of comments for git diffs, that the `lineno` used here belongs to the file in the version that belongs to `version_id` and not it's parent. Can be ``null`` in case a comment belongs to the whole file and not to a specific line. - :>json int version_id: The id of the version. - :>json int user.id: The id for an author. - :>json string user.name: The name for an author. - :>json string user.username: The username for an author. - :>json string|null user.url: The link to the profile page for an author, if the author's profile is public. - -.. http:post:: /api/v5/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/ - - Create a draft comment for a specific version. - - :`. - -.. http:delete:: /api/v5/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/(int:comment_id)/ - - Delete a draft comment. - - :statuscode 204: The comment has been deleted successfully. - :statuscode 404: The user doesn't have the permission to delete. This might happen when someone tries to delete a comment created by another reviewer or author. - - -.. http:patch:: /api/v5/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/(int:comment_id) - - Update a comment, it's filename or line number. - - :`. diff --git a/docs/topics/api/v3_legacy/addons.rst b/docs/topics/api/v3_legacy/addons.rst index eb0d12ccffb6..fa6774fca51b 100644 --- a/docs/topics/api/v3_legacy/addons.rst +++ b/docs/topics/api/v3_legacy/addons.rst @@ -456,7 +456,7 @@ Replacement Add-ons .. _v3-addon-replacement-addons: -This endpoint returns a list of suggested replacements for legacy add-ons that are unsupported in Firefox 57. Added to support the TAAR recommendation service. +This endpoint returns a list of suggested replacements for legacy add-ons that are unsupported in Firefox 57. .. http:get:: /api/v3/addons/replacement-addon/ @@ -508,16 +508,15 @@ Recommendations .. _v3-addon-recommendations: -This endpoint provides recommendations of other addons to install, fetched from the `recommendation service `_. -Four recommendations are fetched, but only valid, publicly available addons are shown (so max 4 will be returned, and possibly less). +This endpoint provides recommendations of other addons to install. Maximum four recommendations will be returned. .. http:get:: /api/v3/addons/recommendations/ :query string guid: Fetch recommendations for this add-on guid. :query string lang: Activate translations in the specific language for that query. (See :ref:`translated fields `) - :query boolean recommended: Fetch recommendations from the recommendation service, or return a curated fallback list instead. - :>json string outcome: Outcome of the response returned. Will be either: ``recommended`` - responses from recommendation service; ``recommended_fallback`` - service timed out or returned empty results so we returned fallback; ``curated`` - ``recommended=False`` was requested so fallback returned. - :>json string|null fallback_reason: if ``outcome`` was ``recommended_fallback`` then the reason why. Will be either: ``timeout`` or ``no_results``. + :query boolean recommended: Ignored. + :>json string outcome: Outcome of the response returned. Will always be ``curated``. + :>json null fallback_reason: Always null. :>json int count: The number of results for this query. :>json string next: The URL of the next page of results. :>json string previous: The URL of the previous page of results. diff --git a/docs/topics/api/v3_legacy/discovery.rst b/docs/topics/api/v3_legacy/discovery.rst index 3d3b467b373d..c990fbaf6380 100644 --- a/docs/topics/api/v3_legacy/discovery.rst +++ b/docs/topics/api/v3_legacy/discovery.rst @@ -25,7 +25,7 @@ Firefox (about:addons). :>json array results: The array containing the results for this query. :>json string results[].heading: The heading for this item. May contain some HTML tags. :>json string|null results[].description: The description for this item, if any. May contain some HTML tags. - :>json boolean results[].is_recommendation: If this item was from the recommendation service, rather than static curated content. + :>json boolean results[].is_recommendation: Always False :>json object results[].addon: The :ref:`add-on ` for this item. Only a subset of fields are present: ``id``, ``current_version`` (with only the ``compatibility`` and ``files`` fields present), ``guid``, ``icon_url``, ``name``, ``slug``, ``theme_data``, ``type`` and ``url``. @@ -35,25 +35,21 @@ Discovery Recommendations .. _v3-disco-recommendations: -If a telemetry client id is passed as a parameter to the discovery pane api -endpoint then static curated content is amended with recommendations from the -`recommendation service `_. The same number -of results will be returned as a standard discovery response and only extensions -(not themes) are recommended. Only valid, publicly available addons are shown. - E.g. a standard discovery pane will display 7 items, 4 extensions and 3 themes. Up to 4 extensions will be replaced with recommendations; the 3 themes will not be replaced. The API will still return a total of 7 items. .. note:: + A standard discovery pane will display 7 items, 4 extensions and 3 themes. + Specifying an ``edition`` parameter disables recommendations - the ``telemetry-client-id`` is ignored and standard discovery response returned. .. http:get:: /api/v3/discovery/?telemetry-client-id=12345678-90ab-cdef-1234-567890abcdef - :query string telemetry-client-id: The telemetry client ID to be passed to the TAAR service. - :query string lang: In addition to activating translations (see :ref:`Discovery Content `), this will be passed as `locale` to TAAR. - :query string platform: The platform identifier to be passed to TAAR. - :query string branch: Additional parameter passed along to TAAR. - :query string study: Additional parameter passed along to TAAR. + :query string telemetry-client-id: Ignored. Was previously an optional sha256 hash of the telemetry client ID to be passed to the TAAR service to enable recommendations. + :query string lang: Activates translations (see :ref:`Discovery Content `). + :query string platform: Ignored. + :query string branch: Ignored. + :query string study: Ignnored. diff --git a/docs/topics/api/v4_frozen/addons.rst b/docs/topics/api/v4_frozen/addons.rst index 0cd6702b7ee3..9be0bdbd5bfc 100644 --- a/docs/topics/api/v4_frozen/addons.rst +++ b/docs/topics/api/v4_frozen/addons.rst @@ -270,14 +270,12 @@ This endpoint allows you to fetch a specific add-on by id, slug or guid. ============== ========================================================== line "By Firefox" category recommended Recommended category - sponsored Sponsored category spotlight Spotlight category strategic Strategic category - verified Verified category badged A meta category that's available for the ``promoted`` search filter that is all the categories we expect an API client to expose as "reviewed" by Mozilla. - Currently equal to ``line&recommended&sponsored&verified``. + Currently equal to ``line&recommended``. ============== ========================================================== ----------------------------- @@ -459,7 +457,7 @@ Replacement Add-ons .. _v4-addon-replacement-addons: -This endpoint returns a list of suggested replacements for legacy add-ons that are unsupported in Firefox 57. Added to support the TAAR recommendation service. +This endpoint returns a list of suggested replacements for legacy add-ons that are unsupported in Firefox 57. .. http:get:: /api/v4/addons/replacement-addon/ @@ -479,17 +477,15 @@ Recommendations .. _v4-addon-recommendations: -This endpoint provides recommendations of other addons to install, fetched from the `recommendation service `_. -Four recommendations are fetched, but only valid, publicly available addons are shown (so max 4 will be returned, and possibly less). +This endpoint provides recommendations of other addons to install. Maximum four recommendations will be returned. .. http:get:: /api/v4/addons/recommendations/ :query string guid: Fetch recommendations for this add-on guid. :query string lang: Activate translations in the specific language for that query. (See :ref:`translated fields `) - :query boolean recommended: Fetch recommendations from the recommendation service, or return a curated fallback list instead. - :>json string outcome: Outcome of the response returned. Will be either: ``recommended`` - responses from recommendation service; ``recommended_fallback`` - service timed out or returned empty or invalid results so we returned fallback; ``curated`` - ``recommended=False`` was requested so fallback returned. - :>json string|null fallback_reason: if ``outcome`` was ``recommended_fallback`` then the reason why. Will be either: ``timeout``, ``no_results``, or ``invalid_results``. - :>json int count: The number of results for this query. + :query boolean recommended: Ignored. + :>json string outcome: Outcome of the response returned. Will always be ``curated``. + :>json null fallback_reason: Always null.:>json int count: The number of results for this query. :>json string next: The URL of the next page of results. :>json string previous: The URL of the previous page of results. :>json array results: An array of :ref:`add-ons `. The following fields are omitted for performance reasons: ``release_notes`` and ``license`` fields on ``current_version`` and ``current_beta_version``, as well as ``picture_url`` from ``authors``. diff --git a/docs/topics/api/v4_frozen/discovery.rst b/docs/topics/api/v4_frozen/discovery.rst index 9efd186d32c0..5e8229c07d84 100644 --- a/docs/topics/api/v4_frozen/discovery.rst +++ b/docs/topics/api/v4_frozen/discovery.rst @@ -21,25 +21,17 @@ Firefox (about:addons). .. note:: - If a telemetry client id is passed as a parameter to the discovery pane api - endpoint then static curated content is amended with recommendations from the - `recommendation service `_. The same number - of results will be returned as a standard discovery response and only extensions - (not themes) are recommended. Only valid, publicly available addons are shown. - - E.g. a standard discovery pane will display 7 items, 4 extensions and 3 themes. - Up to 4 extensions will be replaced with recommendations; the 3 themes will not - be replaced. The API will still return a total of 7 items. + A standard discovery pane will display 7 items, 4 extensions and 3 themes. .. http:get:: /api/v4/discovery/ - :query string lang: Activate translations in the specific language for that query. (See :ref:`translated fields `) + :query string lang: Activates translations. (See :ref:`translated fields `) :query string edition: Optionally return content for a specific edition of Firefox. Currently only ``china`` (and the alias ``MozillaOnline``) is supported. - :query string telemetry-client-id: Optional sha256 hash of the telemetry client ID to be passed to the TAAR service to enable recommendations. Must be the hex value of a sha256 hash, otherwise it will be ignored. + :query string telemetry-client-id: Ignored. Was previously an optional sha256 hash of the telemetry client ID to be passed to the TAAR service to enable recommendations. :>json int count: The number of results for this query. :>json array results: The array containing the results for this query. :>json string|null results[].description_text: The description for this item, if any. Text-only, content might slightly differ from ``description`` because of that. - :>json boolean results[].is_recommendation: If this item was from the recommendation service, rather than static curated content. + :>json boolean results[].is_recommendation: Always False. :>json object results[].addon: The :ref:`add-on ` for this item. Only a subset of fields are present: ``id``, ``authors``, ``average_daily_users``, ``current_version`` (with only the ``id``, ``compatibility``, ``is_strict_compatibility_enabled`` and ``files`` fields present), ``guid``, ``icon_url``, ``name``, ``ratings``, ``previews``, ``slug``, ``theme_data``, ``type`` and ``url``. @@ -51,8 +43,7 @@ Editorial Content This endpoint allows you to fetch all editorial content for Discovery Pane Recommendations. This is used internally to generate .po files containing the -strings defined by the content team. It is also used by TAAR service to obtain a list -of appropriate add-ons to recommended. +strings defined by the content team. .. http:get:: /api/v4/discovery/editorial/ diff --git a/docs/topics/api/v4_frozen/reviewers.rst b/docs/topics/api/v4_frozen/reviewers.rst index b8060dc40334..163f1e51a4a1 100644 --- a/docs/topics/api/v4_frozen/reviewers.rst +++ b/docs/topics/api/v4_frozen/reviewers.rst @@ -129,304 +129,3 @@ denied. :statuscode 202: Success. :statuscode 409: The add-on GUID was already denied. - -------------- -List Versions -------------- - -This endpoint allows you to list versions that can be used either for :ref:`browsing ` or diffing versions. - - .. note:: - Requires authentication and the current user to have ``ReviewerTools:View`` - permission for listed add-ons as well as ``Addons:ReviewUnlisted`` for - unlisted add-ons. Additionally the current user can also be the owner - of the add-on. - - This endpoint is not paginated as normal, and instead will return all - results without obeying regular pagination parameters. - - -If the user doesn't have ``AddonsReviewUnlisted`` permissions only listed versions are shown. Otherwise it can contain mixed listed and unlisted versions. - -.. http:get:: /api/v4/reviewers/addon/(int:addon_id)/versions/ - - :>json int id: The version id. - :>json string channel: The version channel, which determines its visibility on the site. Can be either ``unlisted`` or ``listed``. - :>json string version: The version number string for the version. - -.. _v4-reviewers-versions-browse: - ------- -Browse ------- - -This endpoint allows you to browse through the contents of an Add-on version. - - .. note:: - Requires authentication and the current user to have ``ReviewerTools:View`` - permission for listed add-ons as well as ``Addons:ReviewUnlisted`` for - unlisted add-ons. Additionally the current user can also be the owner - of the add-on. - -.. http:get:: /api/v4/reviewers/addon/(int:addon_id)/versions/(int:version_id)/ - - Inherits the following properties from :ref:`version detail `: ``id``, ``channel``, ``reviewed`` and ``version``. - - .. _v4-reviewers-versions-browse-detail: - - :param string file: The specific file in the XPI to retrieve. Defaults to manifest.json, install.rdf or package.json for Add-ons as well as the XML file for search engines. - :param boolean file_only: Indicates that the API should only return data for the requested file, and not version data. If this is ``true`` then the only property returned of those listed below is the ``file`` property. - :>json string validation_url_json: The absolute url to the addons-linter validation report, rendered as JSON. - :>json string validation_url: The absolute url to the addons-linter validation report, rendered as HTML. - :>json boolean has_been_validated: ``True`` if the version has been validated through addons-linter. - :>json object addon: A simplified :ref:`add-on ` object that contains only a few properties: ``id``, ``name``, ``icon_url`` and ``slug``. - :>json array file_entries[]: The complete file-tree of the extracted XPI. - :>json int file_entries[].depth: Level of folder-tree depth, starting with 0. - :>json string file_entries[].filename: The filename of the file. - :>json string file_entries[].path: The absolute path (from the root of the XPI) of the file. - :>json string file_entries[].mime_category: The mime type category of this file. Can be ``image``, ``directory``, ``text`` or ``binary``. - :>json object file: The requested file. - :>json int file.id: The id of the submitted file (i.e., the xpi file). - :>json string file.content: Raw content of the requested file. - :>json string file.selected_file: The selected file, either from the ``file`` parameter or the default (manifest.json, install.rdf or package.json for Add-ons as well as the XML file for search engines). - :>json string|null file.download_url: The download url of the selected file or ``null`` in case of a directory. - :>json string file.mimetype: The determined mimetype of the selected file or ``application/octet-stream`` if none could be determined. - :>json string file.sha256: SHA256 hash of the selected file. - :>json int file.size: The size of the selected file in bytes. - :>json string file.filename: The filename of the file. - :>json string file.mime_category: The mime type category of this file. Can be ``image``, ``directory``, ``text`` or ``binary``. - :>json boolean uses_unknown_minified_code: Indicates that the selected file could be using minified code. - - -------- -Compare -------- - -This endpoint allows you to compare two Add-on versions with each other. - - .. note:: - Requires authentication and the current user to have ``ReviewerTools:View`` - permission for listed add-ons as well as ``Addons:ReviewUnlisted`` for - unlisted add-ons. Additionally the current user can also be the owner - of the add-on. - -.. http:get:: /api/v4/reviewers/addon/(int:addon_id)/versions/(int:base_version_id)/compare_to/(int:version_id)/ - - .. note:: - - Contrary to what ``git diff`` does, this API renders a hunk full of unmodified lines for unmodified files. - - Inherits most properties from :ref:`browse detail `, except that most of the `file.entries[]` properties - and `file.download_url` can be `null` in case of a deleted file. - - Properties specific to this endpoint: - - :>json array file_entries[]: The complete file-tree of the extracted XPI. - :>json string file.entries[].status: Status of this file, see https://git-scm.com/docs/git-status#_short_format - :>json int file_entries[].depth: Level of folder-tree depth, starting with 0. - :>json string file_entries[].filename: The filename of the file. - :>json string file_entries[].path: The absolute path (from the root of the XPI) of the file. - :>json string file_entries[].mime_category: The mime type category of this file. Can be ``image``, ``directory``, ``text`` or ``binary``. - :>json object|null diff: See the following output with inline comments for a complete description. - :>json object base_file: The file attached to the base version you're comparing against. - :>json object base_file.id: The id of the base file. - :>json boolean uses_unknown_minified_code: Indicates that the selected file in either the current or the parent version could be using minified code. - - Git patch we're talking about: - - .. code:: diff - - diff --git a/README.md b/README.md - index a37979d..b12683c 100644 - --- a/README.md - +++ b/README.md - @@ -1 +1 @@ - -# beastify - +Updated readme - diff --git a/manifest.json b/manifest.json - index aba695f..24f385f 100644 - --- a/manifest.json - +++ b/manifest.json - @@ -1,36 +1 @@ - -{ - - - - "manifest_version": 2, - - "name": "Beastify", - - "version": "1.0", - - - - "permissions": [ - - "http://*/*", - - "https://*/*", - - "bookmarks", - - "made up permission", - - "https://google.com/" - - ], - - - - "content_scripts": [ - - { - - "matches": ["*://*.mozilla.org/*"], - - "js": ["borderify.js"] - - }, - - { - - "matches": ["*://*.mozilla.com/*", "https://*.mozillians.org/*"], - - "js": ["borderify.js"] - - } - - ], - - - - "browser_action": { - - "default_icon": "button/beasts.png", - - "default_title": "Beastify", - - "default_popup": "popup/choose_beast.html" - - }, - - - - "web_accessible_resources": [ - - "beasts/*.jpg" - - ] - - - -} - +{"id": "random"} - - - The following represents the git patch from above. - - .. code:: javascript - - "diff": { - "path": "README.md", - "old_path": "README.md", - "size": 15, // Size in bytes - "lines_added": 1, // How many lines got added - "lines_deleted": 1, // How many lines got deleted - "is_binary": false, // Is this a binary file (as determined by git) - "mode": "M", // Status of this file, see https://git-scm.com/docs/git-status#_short_format - "hunks": [ - { - "header": "@@ -1 +1 @@\\n", - "old_start": 1, - "new_start": 1, - "old_lines": 1, - "new_lines": 1, - "changes": [ - { - "content": "# beastify\\n", - "type": "delete", - "old_line_number": 1, - "new_line_number": -1 - }, - { - "content": "Updated readme\\n", - "type": "insert", - "old_line_number": -1, - "new_line_number": 1 - } - ] - } - ], - "parent": "075c5755198be472522477a1b396951b3b68ac18", - "hash": "00161dcf22afb7bab23cf205f0c903eb5aad5431" - } - - ----------------- -Canned Responses ----------------- - -This endpoint allows you to retrieve a list of canned responses. - - .. note:: - Requires authentication and the current user to have any - reviewer-related permission. - -.. http:get:: /api/v4/reviewers/canned-responses/ - - .. _v4-reviewers-canned-response-detail: - - Retrieve canned responses - - .. note:: - Because this endpoint is not returning too much data it is not - paginated as normal, and instead will return all results without - obeying regular pagination parameters. - - :>json int id: The canned response id. - :>json string title: The title of the canned response. - :>json string response: The text that will be filled in as the response. - :>json string category: The category of the canned response. For example, "Other", "Privacy reasons" etc. - - ------------------ -Drafting Comments ------------------ - -These endpoints allow you to draft comments that can be submitted through the regular reviewer pages. - - .. note:: - Requires authentication and the current user to have ``ReviewerTools:View`` - permission for listed add-ons as well as ``Addons:ReviewUnlisted`` for - unlisted add-ons. Additionally the current user can also be the owner - of the add-on. - - -.. http:get:: /api/v4/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/ - - Retrieve existing draft comments for a specific version. See :ref:`pagination ` for more details. - - :>json int count: The number of comments for this version. - :>json string next: The URL of the next page of results. - :>json string previous: The URL of the previous page of results. - :>json array results: An array of :ref:`comments `. - - -.. http:get:: /api/v4/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/(int:comment_id)/ - - .. _v4-reviewers-draft-comment-detail-object: - - :>json int id: The id of the draft comment object. - :>json string comment: The comment that is being drafted as part of a review. Specific to a line in a file. - :>json string|null filename: The full file path a specific comment is related to. Can be ``null`` in case a comment doesn't belong to a specific file but the whole version. - :>json int|null lineno: The line number a specific comment is related to. Please make sure that in case of comments for git diffs, that the `lineno` used here belongs to the file in the version that belongs to `version_id` and not it's parent. Can be ``null`` in case a comment belongs to the whole file and not to a specific line. - :>json int version_id: The id of the version. - :>json int user.id: The id for an author. - :>json string user.name: The name for an author. - :>json string user.username: The username for an author. - :>json string|null user.url: The link to the profile page for an author, if the author's profile is public. - :>json object|null canned_response: Object holding the :ref:`canned response ` if set. - -.. http:post:: /api/v4/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/ - - Create a draft comment for a specific version. - - :` (optional). - - :statuscode 201: New comment has been created. - :statuscode 400: An error occurred, check the `error` value in the JSON. - :statuscode 403: The user doesn't have the permission to create a comment. This might happen (among other cases) when someone without permissions for unlisted versions tries to add a comment for an unlisted version (which shouldn't happen as the user doesn't see unlisted versions, but it's blocked here too). - - **Response** - In case of successful creation, the response is a :ref:`draft comment object`. - -.. http:delete:: /api/v4/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/(int:comment_id)/ - - Delete a draft comment. - - :statuscode 204: The comment has been deleted successfully. - :statuscode 404: The user doesn't have the permission to delete. This might happen when someone tries to delete a comment created by another reviewer or author. - - -.. http:patch:: /api/v4/reviewers/addon/(int:addon_id)/versions/(int:version_id)/draft_comments/(int:comment_id) - - Update a comment, it's filename or line number. - - :` (optional). - - :statuscode 200: The comment has been updated. - :statuscode 400: An error occurred, check the `error` value in the JSON. - - **Response** - In case of successful creation, the response is a :ref:`draft comment object`. diff --git a/docs/topics/blocklist.md b/docs/topics/blocklist.md new file mode 100644 index 000000000000..1493bb839b7a --- /dev/null +++ b/docs/topics/blocklist.md @@ -0,0 +1,192 @@ +# AMO Blocklist + +(blocklist-doc)= + +This is a high-level overview of the addons-server implementation of the addons blocklist. + +## Full-Stack Overview + +(blocklist-doc-overview)= + +Firefox determines which add-ons are unsafe to allow to continue to be enabled by checking a blocklist: + +- With v1 and v2 blocklist this is literally a list of addon guids, plus other metadata, that should be blocked from executing (v1 is XML format, v2 is JSON format); +- With v3 blocklist this is a bloomfilter that is queried - if the addon guid and xpi version is present then it's in the blocklist so should be blocked by Firefox. The v3 system supports both **Hard Blocks** (add-on is completely disabled) and **Soft Blocks** (add-on is disabled but user can re-enable). + +**The blocklists are all served via Firefox Remote Settings (the current implementation is Kinto):** + + + +- the v3 blocklist bloomfilter files are attachments in the records of +- the v2 blocklist is the full output of +- the v1 blocklist is a (server-side) wrapper around the v2 blocklist that rewrites the JSON into XML + +```{admonition} legacy +v2/v1 are referred to as "legacy" blocklist in these docs. +``` + +AMO holds the addon blocklist records (`Block`) defining blocks for the **v3 system** (including hard/soft type) and generates the v3 bloomfilters and stashes as needed, which are then uploaded to the `addons-bloomfilters` collection on Remote Settings. The `Block` records are managed via the admin tools on addons-server. + +```{admonition} legacy +If any changes are needed to the contents of the legacy blocklist it must be made via the Firefox Remote Settings web admin tool - there is no longer any way to import or export changes between the legacy blocklist and the v3 blocklist. +``` + +## Admin Tool + +(blocklist-doc-admin)= + +_Block_ records aren't created and changed directly via the admin tool; instead _BlockSubmission_ records are created that hold details of the submission of (potentially many) blocks that will be created, updated, or deleted. +If the add-ons that the Block affects are used by a significant number of users (see _DUAL_SIGNOFF_AVERAGE_DAILY_USERS_THRESHOLD_ setting - currently 100k) then the BlockSubmission must be signed off (approved) by another admin user first. + +An admin user can select a block type, addon and a number of versions to block, or all versions. + +Once the submission is approved - or immediately after saving if the average daily user counts are under the threshold - a task is started to asynchronously create, update, or delete, Block records. + +A block can be hardened or softened via the admin tool which results in a new BlockSubmission record. + +Ultimately, Each "block" in the blocklist is a unique pair of the addon guid and the version string allowing for fine grained control over which versions of an add-on are blocked or soft blocked or not blocked. + +## Bloomfilter Generation + +(blocklist-doc-bloomfilter)= + +Generating a bloomfilter for the **v3 blocklist** can be quite slow, so a new one is only generated every 6 hours via a cron job (`upload_mlbf_to_remote_settings`) handling **both hard and soft blocks**. The cron job checks if an update is needed by comparing the current state (hard-blocked, soft-blocked, not-blocked items) against the previously generated filter/stash and the base filter; it skips generation if no filter or stash upload is deemed necessary based on this comparison (see {ref}`Process ` below). + +An ad-hoc bloomfilter can be created with the _export_blocklist_ command but it isn't considered for the cron job (or {ref}`stashing `) + +### Bloomfilter records + +(blocklist-doc-bloomfilter-records)= + +For the **v3 blocklist**, **two** base filter records are created on Remote Settings in the `addons-bloomfilters` collection (one for hard blocks, one for soft blocks, distinguished by `attachment_type`) and the filter file (`filter.bin`) uploaded as an attachment to each. The _generation_time_ property represents the point in time when all previous addon guid + versions and blocks were used to generate the bloomfilter. +An add-on version/file from before this time will definitely be accounted for in the bloomfilter so we can reliably assert if it's blocked or not. +An add-on version/file from after this time can't be reliably asserted - there may be false positives or false negatives. + +See . + +#### Bloomfilter record example + +```json +{ + "schema": 1740062666739, + "attachment": { + "hash": "37ba24caec49afe6c97c424623e226e31ad052286ffa66d794eb82497dabc279", + "size": 28561, + "filename": "filter.bin", + "location": "staging/addons-bloomfilters/1234567890.bin", + "mimetype": "application/octet-stream" + }, + "key_format": "{guid}:{version}", + "attachment_type": "bloomfilter-base", + "generation_time": 1587990908999, + "id": "851e162f-1227-486f-9abe-19bfdcb9fd12", + "last_modified": 1740063264110 + } +``` + +_(A similar record exists with `attachment_type: "softblocks-bloomfilter-base"` for the soft block base filter)_ + +### Stashing + +(blocklist-doc-stashing)= + +Because the bloomfilter files can be quite large, for the **v3 blocklist**, "stash" files are also generated when changes are small compared to the base filters. These represent the changes (for **both hard and soft blocks**) since the _previous_ bloomfilter generation and can be used by Firefox v3 clients instead of downloading a full new filter, saving on bandwidth. + +Multiple stashes can be applied by Firefox (in chronological order using `stash_time`) to match the state of an up-to-date bloomfilter. + +The v3 stash data includes: + +- `blocked`: New hard blocks since the previous generation. +- `unblocked`: Items no longer hard or soft blocked since the previous generation. +- `soft_blocked`: New soft blocks since the previous generation. + +**Important:** When new base filters are uploaded (due to significant changes compared to the _base_, a missing base, or `force_base` being used), _all existing stash records are deleted_ from the Remote Settings collection. Clients fetching updates after this point will need to download the new base filters, and stashing effectively restarts for v3 clients. + +Stash data is stored directly within the Remote Settings record JSON, not as an attachment. + +#### Stash record example (v3) + +```json +{ + "id": "example-stash-record-67890", + "last_modified": 1678887000000, + "key_format": "{guid}:{version}", + "stash_time": 1678887000123, // Example Timestamp (milliseconds) + "stash": { + "blocked": [ + "{6f6b1eaa-bb69-4cdb-a24f-1014493d4290}:10.48", + "kittens@pioneer.mozilla.com:1.2", + "kittens@pioneer.mozilla.com:1.1", + "{b01e0601-eddc-4306-886b-8a4fb5c38a1e}:1", + "{232f11df-20ca-49d4-94eb-e3e63d7ae773}:1.1.2", + "kittens@pioneer.mozilla.com:1.3", + ], + "unblocked": [ + "{896aff0b-d86e-4dd5-9097-5869579b4c28}:1.2", + "{95ffc924-6ea7-4dfb-8f7b-1dd44f2159d1}:1.22.2" + ] + } +} +``` + +The blocked/soft_blocked items represent new versions that should be blocked or soft blocked in addition to any matches in the bloomfilter; the unblocked items represent versions that shouldn't be blocked or soft blocked (even though they would match the bloomfilter). _stash_time_ is a timestamp that can be relied on to order the stashes. + +If a version transitions from one block type to another in a stash, from a logical standpoint two operations are happening: + +- It is ``unblocked`` from the previous block type. +- It is ``blocked`` or ``soft_blocked`` for the new block type. + +We explicitly remove it from the "unblocked" list in this case to prevent double counting the same key in a single stash record. Firefox will use the first matching stash state, but this reduces the size of the stash and guarantees only one possible state per key at a given time. + +### addons-bloomfilter collection + +(blocklist-doc-collection)= + +For the **v3 blocklist**, the `addons-bloomfilters` collection on Remote Settings will contain: + +- Two attachment records, each with an `attachment_type` "bloomfilter-base" or "softblocks-bloomfilter-base" which are the base bloomfilters to compare the stash files to. +- Zero or more stash records, each with stash data directly in the data property. + +The client-side algorithm for **v3 blocklist consumers** would be to: + +- Get the entire collection from Remote Settings (the implementation supports diffing so only new records would be downloaded). +- Download the **two** base bloomfilter attachments (`bloomfilter-base` and `softblocks-bloomfilter-base`) if it hasn't already been downloaded. +- Gather the stash records and then sort them by "newest first". +- Look up the installed addons by `guid:version`, stopping at the first matching blocks state, either blocked, softblocked or unblocked (**in this order**). Assuming a block was found in the stashes, if the stash_time is older than the MLBF generation_time, Firefox looks up the key in the MLBF. The hard BF is checked first, then the soft BF. + +#### Stashing support disabled in Firefox + +If stashing support is disabled in a Firefox version the stash records can be ignored and the **two latest base filters** (`bloomfilter-base` and `softblocks-bloomfilter-base`) will be used instead. (Records with a bloomfilter attachment always have a _generation_time_ field). Firefox would just download the latest attachment and use that as it's bloomfilter. + +### Process + +(blocklist-doc-process)= + +**The server process performed by the cron job (`upload_mlbf_to_remote_settings`) for the v3 blocklist is:** + +- If the `blocklist_mlbf_submit` waffle switch is enabled (or bypassed), proceed. Otherwise, stop. +- Generate the current lists of hard-blocked, soft-blocked, and not-blocked `{guid}:{version}` items based on signed add-ons and `Block` records. +- Load the state (item lists) from the _previous_ successful generation and the _current base filters_ from storage/config. +- Compare current state to previous/base states to decide action: + - **Filter Upload?** If changes for _either_ hard or soft blocks vs. their respective _base filter_ exceed a threshold (default 5000), or `force_base` is true, or a base filter is missing, flag for a filter upload. + - **Stash Upload?** If _not_ uploading filters, but there are _any_ changes for _either_ hard or soft blocks vs. the _previous generation_, flag for a stash upload. + - **Skip?** If neither flag is set, stop the process. +- Generate required files based on the decision: + - Filter Upload: Generate _two_ separate bloomfilter files (`filter-blocked`, `filter-soft_blocked`). + - Stash Upload: Generate _one_ unified `stash.json` file with diffs vs. the previous generation. +- Trigger the asynchronous `upload_filter` task, passing the generation timestamp and required actions (e.g., `UPLOAD_BLOCKED_FILTER`, `UPLOAD_SOFT_BLOCKED_FILTER`, `CLEAR_STASH`, `UPLOAD_STASH`). +- The `upload_filter` task then executes: + - Connects to the `addons-bloomfilters` collection on Remote Settings. + - If uploading filters: Uploads the two new filter attachments (hard=`bloomfilter-base`, soft=`softblocks-bloomfilter-base`), deletes any previous filter attachments of the same types, and **deletes all existing stash records**. + - If uploading a stash: Uploads the new stash JSON data in a record. + - Commits the Remote Settings changes. + - Updates AMO config keys with the new timestamp(s). + - Triggers cleanup of old local files. + +## Legacy Blocklist + +(blocklist-doc-legacy)= + +To populate the blocklist on AMO the legacy blocklist on Remote Settings was imported; all guids that matched addons on AMO (and that had at least one webextension version) were added; any guids that were regular expressions were "expanded" to individual records for each addon present in the AMO database. + +Support for importing the legacy blocklist into AMO, and exporting changes from AMO into the legacy blocklist, has now been removed; it is no longer possible to propagate changes made to the v2 blocklist via the remote-settings web admin tool to the v3 blocklist held on AMO, or visa versa. diff --git a/docs/topics/blocklist.rst b/docs/topics/blocklist.rst deleted file mode 100644 index 779bcc469b9e..000000000000 --- a/docs/topics/blocklist.rst +++ /dev/null @@ -1,174 +0,0 @@ -############# -AMO Blocklist -############# - -.. _blocklist-doc: - -This is a high-level overview of the addons-server implementation of the addons blocklist. - - -=================== -Full-Stack Overview -=================== - -.. _blocklist-doc-overview: - -Firefox determines which add-ons are unsafe to allow to continue to be enabled by checking a blocklist. -With v1 and v2 blocklist this is literally a list of addon guids, plus other metadata, that should be blocked from executing (v1 is XML format, v2 is JSON format); -with v3 blocklist this is a bloomfilter that is queried - if the addon guid and xpi version is present then it's in the blocklist so should be blocked by Firefox. - -The blocklists are all served via Firefox Remote Settings (the current implementation is Kinto): - - the v3 blocklist bloomfilter files are attachments in the records of https://firefox.settings.services.mozilla.com/v1/buckets/blocklists/collections/addons-bloomfilters/records - - the v2 blocklist is the full output of https://firefox.settings.services.mozilla.com/v1/buckets/blocklists/collections/addons/records - - the v1 blocklist is a (server-side) wrapper around the v2 blocklist that rewrites the JSON into XML - -.. note:: - v2/v1 are referred to as "legacy" blocklist in these docs. - -AMO holds the addon blocklist records and generates the bloomfilters as needed, which are then uploaded to Remote Settings. The records are managed via the admin tools on addons-server. - -If any changes are needed to the contents of the v1/v2 blocklist it must be made via the Firefox Remote Settings web admin tool - there is no longer any way to import or export changes between the v1/v2 blocklist and the v3 blocklist. - - -========== -Admin Tool -========== - -.. _blocklist-doc-admin: - -`Block` records aren't created and changed directly via the admin tool; instead `BlockSubmission` records are created that hold details of the submission of (potentially many) blocks that will be created, updated, or deleted. -If the add-ons that the Block affects are used by a significant number of users (see `DUAL_SIGNOFF_AVERAGE_DAILY_USERS_THRESHOLD` setting - currently 100k) then the BlockSubmission must be signed off (approved) by another admin user first. - -Once the submission is approved - or immediately after saving if the average daily user counts are under the threshold - a task is started to asynchronously create, update, or delete, Block records. - - -====================== -Bloomfilter Generation -====================== - -.. _blocklist-doc-bloomfilter: - -Generating a bloomfilter can be quite slow, so a new one is only generated every 6 hours - or less frequently if no Block records have been changed/added/deleted in that time - via a cron job. - -An ad-hoc bloomfilter can be created with the `export_blocklist` command but it isn't considered for the cron job (or :ref:`stashing `) - -------------------- -Bloomfilter records -------------------- - -.. _blocklist-doc-bloomfilter-records: - -A record is created on Remote Settings for each bloomfilter and the filter uploaded as an attachment. The `generation_time` property represents the point in time when all previous addon guid + versions and blocks were used to generate the bloomfilter. -An add-on version/file from before this time will definitely be accounted for in the bloomfilter so we can reliably assert if it's blocked or not. -An add-on version/file from after this time can't be reliably asserted - there may be false positives or false negatives. - -See https://github.com/mozilla/addons-server/issues/13695 and https://github.com/mozilla/addons-server/blob/master/src/olympia/blocklist/cron.py - - -Bloomfilter record example -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: json - - { - "attachment": { - "hash": "37ba24caec49afe6c97c424623e226e31ad052286ffa66d794eb82497dabc279", - "size": 28561, - "filename": "filter.bin", - "location": "staging/addons-bloomfilters/1234567890.bin", - "mimetype": "application/octet-stream" - }, - "key_format": "{guid}:{version}", - "attachment_type": "bloomfilter-base", - "generation_time": 1587990908999, - } - - --------- -Stashing --------- - -.. _blocklist-doc-stashing: - -Because the bloomfilter files can be quite large "stash" files are also generated, which represent the changes since the previous bloomfilter generation and can be used by Firefox instead to save on bandwidth. - -Multiple stashes can be applied by Firefox (in chronological order) to match the state of an up-to-date bloomfilter. - - -Stash record example -^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: json - - { - "stash": { - "blocked": [ - "{6f6b1eaa-bb69-4cdb-a24f-1014493d4290}:10.48", - "kittens@pioneer.mozilla.com:1.2", - "kittens@pioneer.mozilla.com:1.1", - "{b01e0601-eddc-4306-886b-8a4fb5c38a1e}:1", - "{232f11df-20ca-49d4-94eb-e3e63d7ae773}:1.1.2", - "kittens@pioneer.mozilla.com:1.3", - ], - "unblocked": [ - "{896aff0b-d86e-4dd5-9097-5869579b4c28}:1.2", - "{95ffc924-6ea7-4dfb-8f7b-1dd44f2159d1}:1.22.2" - ] - }, - "key_format": "{guid}:{version}", - "stash_time": 1587990908999, - } - -The blocked items represent new versions that should be blocked in addition to any matches in the bloomfilter; the unblocked items represent versions that shouldn't be blocked (even though they would match the bloomfilter). `stash_time` is a timestamp that can be relied on to order the stashes. - - ------------------------------ -addons-bloomfilter collection ------------------------------ - -.. _blocklist-doc-collection: - -The collection on Remote Settings at any given point will consist of a single record with `"attachment-type": "bloomfilter-base"`, which is the base bloomfilter to compare the stash files to, and potentially subsequent records which either contain an attachment with `"attachment-type": "bloomfilter-full"`, or stash data directly in the data property. The client-side algorithm would be to: - -* Get the entire collection from Remote Settings (the implementation supports diffing so only new records would be downloaded). -* Download the base bloomfilter attachment (`"attachment-type": "bloomfilter-base"`) if it hasn't already been downloaded. -* Gather the stash records and consolidate them, taking into account timestamps so later stashes override earlier stashes. - - -Stashing support disabled in Firefox -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If stashing support is disabled in a Firefox version the stash records can be ignored and all bloomfilters considered instead. (Records with a bloomfilter attachment always have a `generation_time` field). Firefox would just download the latest attachment and use that as it's bloomfilter. - - -------- -Process -------- - -.. _blocklist-doc-process: - -The server process is: - * If the `blocklist_mlbf_submit` waffle switch is enabled, check if there have been any changes to the blocklist since the previous execution of the cron job - if not return without any action. (not blocked guids) - * Produce a list of all "guid:version" combinations of all signed webextension addons/versions in the database. (blocked guids) - * Produce a list of "guid:version" combinations that the Block records cover. Blocks have a minimum and maximum version range - 0 being the minimum, and * meaning infinity, so 0 - * would be all versions of an add-on. - * Create and verify a bloomfilter with these two lists (we use https://github.com/mozilla/filter-cascade/); save the filter file and the two lists (as JSON) - - * Compare list of blocked guids from this execution to the base bloomfilter file. If there have been few changes then write those changes to a stash JSON blob - - #. Upload the stash as JSON data in record - #. Upload the filter as an attachment to a separate record with the type `bloomfilter-full` - * If there have been many changes then: - - #. clear the collection on Remote Settings - #. Upload the filter as an attachment to a separate record with the type `bloomfilter-base` - - -================ -Legacy Blocklist -================ - -.. _blocklist-doc-legacy: - -To populate the blocklist on AMO the legacy blocklist on Remote Settings was imported; all guids that matched addons on AMO (and that had at least one webextension version) were added; any guids that were regular expressions were "expanded" to individual records for each addon present in the AMO database. - -Support for importing the legacy blocklist into AMO, and exporting changes from AMO into the legacy blocklist, has now been removed; it is no longer possible to propagate changes made to the v2 blocklist via the remote-settings web admin tool to the v3 blocklist held on AMO, or visa versa. diff --git a/docs/topics/development/acl.md b/docs/topics/development/acl.md new file mode 100644 index 000000000000..ab64d49b53fa --- /dev/null +++ b/docs/topics/development/acl.md @@ -0,0 +1,35 @@ +(acl)= + +# Access Control Lists + +## How permissions work + +On top of that we use the `access.models.GroupUser` and `Group` to define +what access groups a user is a part of, and each group has `rules` defining +which permissions they grant their members, separated by `,`. + +Permissions that you can use as filters can be either explicit or general. + +For example `Admin:EditAddons` means only someone with that permission will +validate. + +If you simply require that a user has _some_ permission in the _Admin_ group +you can use `Admin:%`. The `%` means "any." + +Similarly a user might be in a group that has explicit or general permissions. +They may have `Admin:EditAddons` which means they can see things with that +same permission, or things that require `Admin:%`. + +If a user has a wildcard, they will have more permissions. For example, +`Admin:*` means they have permission to see anything that begins with +`Admin:`. + +The notion of a superuser has a permission of `*:*` and therefore they can +see everything. + +## Django Admin + +Django admin relies on 2 things to gate access: + +- To access the admin itself, `UserProfile.is_staff` needs to be `True`. Our custom implementation allows access to users with a `@mozilla.com` email. +- To access individual modules/apps, `UserProfile.has_perm(perm, obj)` and `UserProfile.has_module_perms(app_label)` need to return `True`. Our custom implementation uses the `Group` of the current user as above, with a mapping constant called `DJANGO_PERMISSIONS_MAPPING` which translates Django-style permissions into our own. diff --git a/docs/topics/development/acl.rst b/docs/topics/development/acl.rst deleted file mode 100644 index 316151c049eb..000000000000 --- a/docs/topics/development/acl.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. _acl: - -==================== -Access Control Lists -==================== - -How permissions work --------------------- - -On top of that we use the ``access.models.GroupUser`` and ``Group`` to define -what access groups a user is a part of, and each group has ``rules`` defining -which permissions they grant their members, separated by ``,``. - -Permissions that you can use as filters can be either explicit or general. - -For example ``Admin:EditAddons`` means only someone with that permission will -validate. - -If you simply require that a user has `some` permission in the `Admin` group -you can use ``Admin:%``. The ``%`` means "any." - -Similarly a user might be in a group that has explicit or general permissions. -They may have ``Admin:EditAddons`` which means they can see things with that -same permission, or things that require ``Admin:%``. - -If a user has a wildcard, they will have more permissions. For example, -``Admin:*`` means they have permission to see anything that begins with -``Admin:``. - -The notion of a superuser has a permission of ``*:*`` and therefore they can -see everything. - - -Django Admin ------------- - -Django admin relies on 2 things to gate access: -- To access the admin itself, ``UserProfile.is_staff`` needs to be ``True``. Our custom implementation allows access to users with a ``@mozilla.com`` email. -- To access individual modules/apps, ``UserProfile.has_perm(perm, obj)`` and ``UserProfile.has_module_perms(app_label)`` need to return ``True``. Our custom implementation uses the ``Group`` of the current user as above, with a mapping constant called ``DJANGO_PERMISSIONS_MAPPING`` which translates Django-style permissions into our own. diff --git a/docs/topics/development/authentication.md b/docs/topics/development/authentication.md new file mode 100644 index 000000000000..5bbe85c5a3b4 --- /dev/null +++ b/docs/topics/development/authentication.md @@ -0,0 +1,42 @@ +# Authentication + +## Firefox Accounts + +Firefox Accounts (FXA) is the authentication system used by addons-server. +In local development and in testing, we default to fake credentials which redirect +to a local fake auth page. If you define real credentials, you will be redirected +to fxa using the specified client id and secret. + +### Local development + +In local development, we default to fake credentials which redirect to a local +fake auth page. Fake credentials of '' are defined by default on the environment and read +into the FXA_CONFIG settings. + +### Production environments + +In production environments, we defined real FXA_CLIENT_ID and FXA_CLIENT_SECRET values +to be used on the corrresponding FXA servers. + +### use_fake_fxa + +A utility method is used to determine if we should use the fake or real fxa redirect. +This function only returns true if the environment is local/test and if the fake fxa client +id and secret are defined. This forces us to use real auth redirection in production environments. + +### Getting real credentials for local development + +You must contact the FxA team to get your own credentials for FxA stage. + +To use these credentials, you can pass them to make up: + +```bash +make up FXA_CLIENT_ID= FXA_CLIENT_SECRET= +``` + +or set them in your environment (on the host machine, not inside the container): + +```bash +export FXA_CLIENT_ID= +export FXA_CLIENT_SECRET= + diff --git a/docs/topics/development/branching.rst b/docs/topics/development/branching.md similarity index 55% rename from docs/topics/development/branching.rst rename to docs/topics/development/branching.md index 097497f6cba0..453ba9c5507f 100644 --- a/docs/topics/development/branching.rst +++ b/docs/topics/development/branching.md @@ -1,15 +1,11 @@ -.. _branching: +(branching)= -================ -Push From Master -================ +# Push From Master -We deploy from the `master`_ branch once a week. If you commit something to master -that needs additional QA time, be sure to use a `waffle`_ feature flag. +We deploy from the [master] branch once a week. If you commit something to master +that needs additional QA time, be sure to use a [waffle] feature flag. - -Local Branches --------------- +## Local Branches Most new code is developed in local one-off branches, usually encompassing one or two patches to fix a bug. Upstream doesn't care how you do local @@ -21,7 +17,7 @@ of related patches from a feature branch. The rule of thumb is to rebase and use fast-forward merge for single patches or a branch of unrelated bug fixes, but to use a merge commit if you have multiple commits that form a cohesive unit. -Here are some tips on `Using topic branches and interactive rebasing effectively `_. +Here are some tips on [Using topic branches and interactive rebasing effectively](http://blog.mozilla.com/webdev/2011/11/21/git-using-topic-branches-and-interactive-rebasing-effectively/). -.. _master: http://github.com/mozilla/addons-server/tree/master -.. _waffle: https://github.com/jsocol/django-waffle +[master]: http://github.com/mozilla/addons-server/tree/master +[waffle]: https://github.com/jsocol/django-waffle diff --git a/docs/topics/development/building_and_running_services.md b/docs/topics/development/building_and_running_services.md new file mode 100644 index 000000000000..9e9efdbe10c3 --- /dev/null +++ b/docs/topics/development/building_and_running_services.md @@ -0,0 +1,181 @@ +# Building and Running Services + +## Dockerfile Details + +The Dockerfile for the **addons-server** project uses a multi-stage build to optimize the image creation process. Here's an overview of the key concepts and design decisions behind it: + +1. **Multi-Stage Build**: + - **Intent**: Multi-stage builds allow Docker to parallelize steps that don't depend on each other and to better cache between layers. This results in more efficient builds by reducing the size of the final image and reusing intermediate layers. + - **Layer Caching**: The use of `--mount=type=cache` arguments helps cache directories across builds, particularly useful for node and pip dependencies, dramatically speeding up future builds. + +2. **OLYMPIA_USER**: + - **Creating a Non-Root User**: The Dockerfile creates an `olympia` user to run the application. This allows the container to run processes as a non-root user, enhancing security by preventing privilege escalation. + - **Why Non-Root?**: Running containers as root is considered an antipattern for Python projects due to security vulnerabilities. Using a non-root user like `olympia` ensures that even if an attacker accesses the container, they cannot escalate privileges to the host. + +3. **Mounts in Docker Compose**: + - **Mounting Local Repository**: The volume `.:/data/olympia` mounts the local Git repository into the container, allowing real-time changes to files within the container. + - **Mounting Dependencies**: The volume `./deps:/data/olympia/deps` mounts the dependencies directory, enabling better caching across builds and providing visibility for debugging directly on the host. + +4. **Environment Variables for OLYMPIA_USER**: + - **Development Setup**: The `OLYMPIA_UID` .env variable is set to the host user ID, ensuring that the container runs with the correct permissions. + +### Best Practices for the Dockerfile + +- **Use as Few Instructions as Possible**: This minimizes the size of the image and reduces build times. +- **Split Long-Running Tasks**: Distinct stages improve caching and concurrency. +- **Prefer `--mount=type=bind` Over `COPY`**: Use bind mounts for files needed for a single command. Bind mounts do not persist data, so modified files will not be in the final layer. +- **Prefer Copying Individual Files Over Directories**: This reduces the likelihood of false cache hits. +- **Use `--mount=type=cache` for Caching**: Cache npm/pip dependencies to speed up builds. +- **Delay Copying Source Files**: This improves cache validity by ensuring that as many stages as possible can be cached. + +## Build Process + +The **addons-server** project uses BuildKit and Bake to streamline the image-building process. + +1. **BuildKit**: + - **Overview**: BuildKit is a modern Docker image builder that enhances performance, scalability, and extensibility. It allows for parallel build steps, caching, and improved efficiency. + - **Driver**: BuildKit uses a driver model to execute builds, with the `docker` driver being the default. + We use the `docker` driver to build our images as it is the fastest and fits the criteria for our project. + We have used the `docker-container` driver in the past, but it is slower due to transferring files in and out of the build container. + The docker driver is slightly slower at rebuilding cached layers but makes up for the difference by building the layers + where they are stored, on the host. + +2. **Bake**: + - **Overview**: Docker Bake is a tool for defining and executing complex build workflows. It simplifies multi-platform builds and allows for more granular control over the build process. + - **Using Bake**: We use Bake to enable building via Docker Compose consistently across local and CI builds. The `build` target in the `docker-compose.yml` file defines the build context and Dockerfile for the addons-server image. + +To build the Docker images for the project, use the following command: + +```sh +make docker_build_web +``` + +This command leverages BuildKit and Bake to efficiently build the required images. +By default buildx will use 2 configuration files: + +- [docker-bake.hcl](../../../docker-bake.hcl) +- .env (Generated by running `make setup` locally) + +Bake will use the .env file to set environment variables in the build process +and the docker-bake.hcl file to define the build process. + +You can specify additional make arguments to control the behaviour of the build: + +- `DOCKER_PUSH` - Push the image to the current registry, defined by the `DOCKER_TARGET` variable in .env. +- `DOCKER_PROGRESS` - Control the build output verbosity. Default is "auto". Set to "plain" for more detailed output. +- `DOCKER_METADATA_FILE` - Specify the file to store build metadata. Default is "buildx-bake-metadata.json". +- `DOCKER_COMMIT` - Set the Git commit hash for the image. If not set, it will be determined automatically. +- `DOCKER_BUILD` - Set a custom build number for the image. If not set, a timestamp will be used. +- `DOCKER_VERSION` - Set a custom version for the image. If not set, it will be determined from Git tags. +- `ARGS` - Pass additional arguments to the `docker buildx bake` command. + +These arguments allow you to customize various aspects of the build process, +from controlling output verbosity to setting specific image metadata. + +When running local builds you can largely ignore these arguments +and just use `make docker_build_web` to build the image. + +### Clearing Cache + +To clear the custom builder cache used for buildkit mount caching: + +```bash +docker builder prune +``` + +Avoid using `docker system prune` as it does not clear the specific builder cache. + +### Docker Ignore + +Our [.dockerignore](../../../.dockerignore) file is used to ignore files and directories that should not be included in the build context. +This is useful to reduce the final image size and speed up the build process. + +Because our image copies all files in the repository to the image, any time even one character in those files changes, +the entire stage is busted and all files are re-copied. This can take 10-30 seconds. Ignoring files that are irrelevant +reduces the number of times this happens and speeds up the build process. + +All files included in the .dockerignore are files we explicitly do not need in production containers. + +Docker ignore is "ignored" during development as we always mount the host repository into the container at runtime. + +> NOTE: Our dockerignore file is a superset of our gitignore file. Any files included in gitignore should also be included in dockerignore. +> dockerignore also includes extra files we do checkin to git but do not need in production containers. E.g .github directory containing github actions files. + +## Managing Containers + +Managing the Docker containers for the **addons-server** project involves using Makefile commands to start, stop, and interact with the services. + +1. **Starting Services**: + - Use `make up` to start the Docker containers: + + ```sh + make up + ``` + + - This command ensures all necessary files are created and the Docker Compose project is running. + +2. **Stopping Services**: + - Use `make down` to stop and remove the Docker containers: + + ```sh + make down + ``` + +3. **Accessing Containers**: + - Access the web container for debugging: + + ```sh + make shell + ``` + + - Access the Django shell within the container: + + ```sh + make djshell + ``` + +4. **Rebuilding Images**: + - Use `make up` to rebuild the Docker images if you make changes to the Dockerfile or dependencies. Remember, `make up` is idempotent, ensuring your image is built and running based on the latest changes. + +This section provides a thorough understanding of the Dockerfile stages, build process using BuildKit and Bake, and commands to manage the Docker containers for the **addons-server** project. For more detailed information on specific commands, refer to the project's Makefile and Docker Compose configuration in the repository. + +## Docker Compose + +We use docker compose under the hood to orchestrate container both locally and in CI. +The `docker-compose.yml` file defines the services, volumes, and networks required for the project. + +Our docker compose project is split into a root [docker-compose.yml](../../../docker-compose.yml) file and additional files for specific environments, +such as [docker-compose.override.yml](../../../docker-compose.override.yml) for CI environments. + +### Healthchecks + +We define healthchecks for the web and worker services to ensure that the containers are healthy and ready to accept traffic. +The health checks ensure the django wsgi server and celery worker node are running and available to accept requests. + +### Environment specific compose files + +- **Local Development**: The `docker-compose.yml` file is used for local development. It defines services like `web`, `db`, `redis`, and `elasticsearch`. +- **Private**: This file includes the customs service that is not open source and should therefore not be included by default. +- **Override**: This file allows modifying the default configuration without changing the main `docker-compose.yml` file. This file is larglely obsolete and should not be used. + +To mount with a specific set of docker compose files you can add the COMPOSE_FILE argument to make up. This will persist your setting to .env. + +```sh +make up COMPOSE_FILE=docker-compose.yml:docker-compose.override.yml +``` + +Files should be separated with a colon. + +### Volumes + +Our project defines volumes to mount and share local data between services. + +- **data_redis,data_elastic,data_rabbitmq**: Used to persist service specific data in a named volume to avoid anonymous volumes in our project. +- **data_mysql**: Used to persist the MySQL data in a named volume to avoid anonymous volumes in our project. +Additionally this volume is "external" to allow the volume to persist across container lifecycle. If you make down, the data will not be destroyed. +- **storage**: Used to persist local media files to nginx. + +We additionally mount serval local directories to the web/worker containers. + +- **.:/data/olympia**: Mounts the local repository into the container to allow real-time changes to files within the container. +- **./deps:/data/olympia/deps**: Mounts the dependencies directory to enable better caching across builds and provide visibility for debugging directly on the host. diff --git a/docs/topics/development/contributing.md b/docs/topics/development/contributing.md new file mode 100644 index 000000000000..23a2e7d2cf20 --- /dev/null +++ b/docs/topics/development/contributing.md @@ -0,0 +1,4 @@ +```{eval-rst} +.. include:: ../../../.github/CONTRIBUTING.md + :parser: myst_parser.sphinx_ +``` diff --git a/docs/topics/development/contributing.rst b/docs/topics/development/contributing.rst deleted file mode 100644 index 65b992f08ead..000000000000 --- a/docs/topics/development/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../.github/CONTRIBUTING.rst diff --git a/docs/topics/development/data_management.md b/docs/topics/development/data_management.md new file mode 100644 index 000000000000..2b5b46d91292 --- /dev/null +++ b/docs/topics/development/data_management.md @@ -0,0 +1,222 @@ +# Data Management + +Effective data management is crucial for the **addons-server** project. This section focuses on how the project handles persistent data, data snapshots, and initial data population. + +## Persistent Data Volumes + +The project uses persistent data volumes to store MySQL data. This ensures that data remains intact even when containers are stopped or removed. For details on how these volumes are defined, refer to the Docker Compose configuration in the repository. + +## External Mounts + +The use of an external mount allows for manual management of the data lifecycle. This ensures that data is preserved even if you run `make down`. By defining the MySQL data volume as external, it decouples the data lifecycle from the container lifecycle, allowing you to manually manage the data. + +## Data Initialization + +When you run `make up` make will run the `initialize` command for you. This command will check if the database exists, and if the elasticsearch index exists. + +If they don't exist it will create them. This command can be run manually as well. + +```sh +make initialize +``` + +This command is responsible for ensuring your local mysql database is migrated, seeded, loaded with data and indexed. +There are a number of different ways to execute this command. In most cases, the default behavior is what you want. +But there are a few additional edge cases that it supports. + +### Clean the database + + ```sh + make initialize INIT_CLEAN=true + ``` + + This will force the database to be recreated, and re-initialized. + +### Load a data backup + + ```sh + make initialize [INIT_LOAD=] + ``` + + This command will load a data backup from a specified path. The optional `INIT_LOAD` argument allows you to + specify the path to the data backup file. If not specified, the initialize command will determine if + data should be loaded based on the current state of the databse, and will load the `_init` data backup. + +### Skip seeding + +```sh +make initialize INIT_SKIP_SEED=true +``` + +This will skip the seeding of the database. This can be useful in CI or if you specifically +want to avoid touching the previous data or creating a new _init backup. + +### Skip index recreation + +```sh +make initialize INIT_SKIP_INDEX=true +``` + +This will skip the recreation of the elasticsearch index. This can be useful in CI or if you specifically +want to avoid touching the previous elasticsearch index. + +```{admonition} Note +If your database is modified significantly and you don't re-index elasticsearch you could end up with +a broken addons-frontend. +``` + +## Data seeding + +`addons-server` uses a a data seeding mechanism to populate the database with the initial data. This data is used to +bootstrap the database with addons and other data to enable development. + +The data seed is treted just like a data backup with a special name `_init`. To recreate the dataseed run: + +```sh +make seed_data +``` + +This will flush the current database, remove the _init backup directory if it exists, run the seed commands, +and finally dump the data back into the _init backup directory. + +The _init backup is used to populate the database with initial data during the initialization process. + +## Data backups + +You can export and load data snapshots to manage data states across different environments or for backup purposes. +The Makefile provides commands to facilitate this. +These commands rely internally on [django-dbbackup](https://django-dbbackup.readthedocs.io/en/stable/) + +- **Data dump**: + + ```sh + make data_dump [ARGS="--name --force"] + ``` + + This command creates a dump of the current MySQL database. The command accepts an optional `name` argument which will determine + the name of the directory created in the backup directory. By default it uses a timestamp to ensure uniqueness. + + You can also specify the `--force` argument to overwrite an existing backup with the same name. + +- **Loading Data**: + + ```sh + make data_load [ARGS="--name "] + ``` + + This command will load data from an existing backup directory, synchronize the storage directory and reindex elasticsearch. + The name is required and must match a directory in the backup directory. + +## Hard Reset Database + +The actual mysql database is created and managed by the `mysqld` container. The database is created on container start +and the actual data is stored in a persistent data volume. This enables data to persist across container restarts. + +`addons-server` assumes that a database named `olympia` already exists and most data management commands will fail +if it does not. + +If you need to hard reset the database (for example, to start with a fresh state), you can use the following command: + +```bash +make down && make docker_mysqld_volume_remove +``` + +This will stop the containers and remove the `mysqld` data volume from docker. The next time you run `make up` it will +create a new empty volume for you and mysql will recreate the database. + +```{admonition} Note +Removing the data volume will remove the actual data! You can and should save a backup before doing this +if you want to keep the data. +``` + +## Migrations + +(squashing-migrations)= +### Squashing migrations + +Squashing migrations makes our app run faster as fewer migration scripts need to be loaded/executed +to get the correct database state. Additionally over time migrations modify the database state. +Once those changes have been made the operations can be suqashed into a smaller set of operations +that can run in a single migration. + +The overall process for squashing migrations should look something like this: + +1) identify a set of mgirations that can be squashed +2) create a migration at the end of the chain that effectively replicates the logic of the squashable set + + This can be done via the management console: + + ```bash + ./manage.py squashmigrations {app_label} {start_number} {end_number} + ``` + + ```{admonition} Important + Do not delete the old migrations in the same patch as squashing. First land the squashed patch. + This should go through CI and be deployed to production first. Then after we have ensured the migration + is registered and run on all of our databases without any problems, then you can delete the old migrations. + ``` + +3) Verify the migrations locally + + Run your migration against a clean master database, this is as close to the production environment as possible locally. On latest master, run: + + ```bash + make up CLEAN_INIT=true + ``` + + ```{admonition} Note + Squashing migrations is not a perfect science. If any of the migrations have custom functions or + other custom logic, you may need to manually adjust the squashed migration to make it work. + ``` + + Run the migrations from a shell: + + ```bash + ./manage.py migrate + ``` + + You should expect no actual changes to the database, because the new migration should not + have made any changes relative to the migrations it is squashing. If the migration script fails + or if any changes are made to the database, something is wrong and you should not proceed. + +4) deploy the app with this new migration to all environments (this ensure the new migration is registered and marked as ran in the various DBs) +5) remove the now irrelevant set of squashable migrations leaving only the new squashed migration + + When deleting old migrations, double check that no other migrations depend on them. + Follow the same testing steps as above to verify the removed migrations do not impact the migration + graph and that migrations can be run successfully without changing the database state. + +#### Visualizing migrations + +Run migration graph from the management console to visualize migraitons for an app. + +```bash +./manage.py migrationgraph {app_label} > graph.md +``` + +This will output some text and a mermaid snippet modeling the migration depdendencies. +Open this with a mermaid previewer/renderer (https://marketplace.cursorapi.com/items?itemName=bierner.markdown-mermaid) + +It will look like this: + +![alt text](../../_static/images/migration_graph_mermaid.png) + +Additionally, migration graphs are published in our model docs and can viewed for each app + +#### Squashing Linear Migrations + +The easiest migrations to squash are linear. They belong to a single app and +each migration in the sequence only depends on the previous migration in the same sequence. + +Linear migrations can be identified most easily using model graph (see above) + +1) Find a linear sequence of migrations that do not have any external dependencies. + + In the example above, notice that addons 0002 through 0009 only depend on each other. + 9 depends on 8, 8 on 7, etc. This is a linear sequence of migrations + that do not have any external dependencies. + +2) Squash the migrations + + Follow steps specified in the [squashing-migrations](#squashing-migrations) section. + diff --git a/docs/topics/development/debugging.rst b/docs/topics/development/debugging.rst deleted file mode 100644 index 8a1356b0fbe9..000000000000 --- a/docs/topics/development/debugging.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. _debugging: - -========= -Debugging -========= - -The :ref:`docker setup ` uses supervisord to run the -django runserver. This means if you want to access the management server -from a shell to run things like ipdb_ you still can. - -Using ipdb ----------- - -As with ipdb normally just add a line in your code at the relevant point: - -.. code-block:: python - - import ipdb; ipdb.set_trace() - -Next connect to the running web container:: - - make debug - -This will bring the Django management server to the foreground and you -can interact with ipdb as you would normally. To quit you can just type -``Ctrl+c``. - -All being well it should look like this:: - - $ make debug - docker exec -t -i olympia_web_1 supervisorctl fg olympia - :/opt/rh/python27/root/usr/lib/python2.7/site-packages/celery/utils/__init__.py:93 - 11:02:08 py.warnings:WARNING /opt/rh/python27/root/usr/lib/python2.7/site-packages/jwt/api_jws.py:118: DeprecationWarning: The verify parameter is deprecated. Please use options instead. - 'Please use options instead.', DeprecationWarning) - :/opt/rh/python27/root/usr/lib/python2.7/site-packages/jwt/api_jws.py:118 - [21/Oct/2015 11:02:08] "PUT /en-US/firefox/api/v4/addons/%40unlisted/versions/0.0.5/ HTTP/1.1" 400 36 - Validating models... - - 0 errors found - October 21, 2015 - 13:52:07 - Django version 1.6.11, using settings 'settings' - Starting development server at http://0.0.0.0:8000/ - Quit the server with CONTROL-C. - [21/Oct/2015 13:57:56] "GET /static/img/app-icons/16/sprite.png HTTP/1.1" 200 3810 - 13:58:01 py.warnings:WARNING /opt/rh/python27/root/usr/lib/python2.7/site-packages/celery/task/sets.py:23: CDeprecationWarning: - celery.task.sets and TaskSet is deprecated and scheduled for removal in - version 4.0. Please use "group" instead (see the Canvas section in the userguide) - - """) - :/opt/rh/python27/root/usr/lib/python2.7/site-packages/celery/utils/__init__.py:93 - > /code/src/olympia/browse/views.py(148)themes() - 147 import ipdb;ipdb.set_trace() - --> 148 TYPE = amo.ADDON_THEME - 149 if category is not None: - - ipdb> n - > /code/src/olympia/browse/views.py(149)themes() - 148 TYPE = amo.ADDON_THEME - --> 149 if category is not None: - 150 q = Category.objects.filter(application=request.APP.id, type=TYPE) - - ipdb> - -Logging -------- - -Logs for the celery and Django processes can be found on your machine in the -`logs` directory. - -Using the Django Debug Toolbar ------------------------------- - -The `Django Debug Toolbar`_ is very powerful and useful when viewing pages from -the website, to check the view used, its parameters, the SQL queries, the -templates rendered and their context. - -To use it please see the official getting started docs: https://django-debug-toolbar.readthedocs.io/en/1.4/installation.html#quick-setup - -.. note:: - - You must know that using the Django Debug Toolbar will slow the website quite a - lot. You can mitigate this by deselecting the checkbox next to the ``SQL`` - panel. - - Also, please note that you should only use the Django Debug Toolbar if you need - it, as it makes CSP report only for your local dev. - -.. note:: - You might have to disable CSP by setting `CSP_REPORT_ONLY = True` in your - local settings because django debug toolbar uses "data:" for its logo, - and it uses "unsafe eval" for some panels like the templates or SQL ones. - -.. _ipdb: https://pypi.python.org/pypi/ipdb -.. _Django Debug Toolbar: http://django-debug-toolbar.readthedocs.io/ diff --git a/docs/topics/development/dependencies.md b/docs/topics/development/dependencies.md deleted file mode 100644 index e6b0c07fc83f..000000000000 --- a/docs/topics/development/dependencies.md +++ /dev/null @@ -1,63 +0,0 @@ -# Project Dependencies - -This document describes how to add/upgrade dependencies in the project. -We use pip to manage dependencies and hashin to lock versions. We use npm to manage frontend dependencies. - -## Python - -### Adding Python Dependencies - -We have 2 requirements files for python dependencies: - -- prod.txt -- dev.txt - -Prod dependencies are used by our django app in runtime. -They are strictly required to be installed in the production environment. - -Dev dependencies are used by our django app in development or by tools we use for linting, testing, etc. - -> If you add just the package name the script will automatically get the latest version for you. - -```bash -hashin -r -``` - -This will add hashes and sort the requirements for you adding comments to -show any package dependencies. - -When it's run check the diff and make edits to fix any issues before -submitting a PR with the additions. - -### Upgrading Python Dependencies - -We mostly rely on dependabot for this. TBD Add more details. - -## Frontend - -### Adding Frontend Dependencies - -We use npm to manage frontend dependencies. To add a new dependency, use the following command: - -```bash -npm install [package]@[version] --save --save-dev -``` - -NPM is a fully featured package manager and so you can use the standard CLI. - -## Updating/Installing dependencies - -To update/install all dependencies, run the following command: - -```bash -make update_deps -``` - -This will install all python and frontend dependencies. It also ensures olympia is installed locally. -By default this command will run in a docker container, but you can run it on a host by targetting the Makefile-docker - -```bash -make -f Makefile-docker update_deps -``` - -This is used in github actions for example that do not need a full container to run. diff --git a/docs/topics/development/dependency_management.md b/docs/topics/development/dependency_management.md new file mode 100644 index 000000000000..483e99c468d1 --- /dev/null +++ b/docs/topics/development/dependency_management.md @@ -0,0 +1,110 @@ +# Dependency Management + +Managing dependencies effectively is crucial for maintaining a stable and consistent development environment. The **addons-server** project uses a well-structured approach to handle Python and Node.js dependencies. + +## Python Dependencies + +Python dependencies are managed using the Makefile and requirements files. All dependencies are installed into the `/data/olympia/deps` directory, which centralizes dependency management and simplifies data mounts. + +- **Environment Variables**: The project sets environment variables for Python CLIs to install dependencies in specific locations. This includes setting paths for `PIP_CACHE_DIR`, `PIP_SRC`, and others to use the `/data/olympia/deps` directory. + +- **Caching Mechanism**: By using Docker build stages, the project isolates the stages responsible for installing dependencies. This prevents these stages from re-running unless the actual dependency files are changed. Additionally, internal Python cache folders are cached, avoiding unnecessary re-downloads of packages and saving time and bandwidth. + +- **Requirements Files**: + - **`pip.txt`**: Specifies the version of pip to guarantee consistency. + - **`prod.txt`**: Lists dependencies used in production deployments. + - **`dev.txt`**: Lists additional dependencies used for development. + +In the Docker build, only production dependencies are included. When running `make up`, the following command is executed to install development dependencies: + +```sh +make docker_extract_deps +``` + +This command installs the development dependencies inside the container, ensuring the development environment is fully set up. + +### Adding Python Dependencies + +We use `hashin` to manage package installs. It helps you manage your `requirements.txt` file by adding hashes to ensure that the installed package versions match your expectations. `hashin` is automatically installed in local developer environments. + +To add a new dependency: + +```bash +hashin -r {requirements} {dependency}=={version} +``` + +This will add hashes and sort the requirements for you, adding comments to show any package dependencies. Check the diff and make edits to fix any issues before submitting a PR with the additions. + +### Managing Python Dependencies + +All Python dependencies are defined in requirements files in the `requirements` directory. Our 3 most important files are: + +- **`pip.txt`**: Specifies the version of pip to guarantee consistency. +- **`prod.txt`**: Dependencies required in the production environment. +- **`dev.txt`**: Dependencies used for development, linting, testing, etc. + +We use Dependabot to automatically create pull requests for updating dependencies. +This is configured in the `.github/dependabot.yml` file targeting files in the `requirements` directory. + +### Managing Transitive Dependencies + +In local development and CI, we install packages using pip with the `--no-deps` flag to prevent pip from installing transitive dependencies. This approach gives us control over the full dependency chain, ensuring reproducible and trustworthy environments. + +## Pip Dependencies + +In order to determine the dependencies a given package requires you can check + +```bash +pip show +``` + +To see the `requirements` field which lists the dependencies. Install missing dependencies manually. + +```{admonition} Note +Ensure to comment in the requirements file above transitive dependencies which direct dependency it is required by. +``` + +## Node.js Dependencies + +Node.js dependencies are managed using npm. + +- **Caching Mechanism**: Node.js dependencies are also cached using Docker build stages. Internal npm cache folders are cached to avoid re-downloading packages unnecessarily. + +### Adding Frontend Dependencies + +To add a new frontend dependency: + +```bash +npm install [package]@[version] --save --save-dev +``` + +NPM is a fully-featured package manager, so you can use the standard CLI. + +## Caching in Docker Build + +The Dockerfile uses build stages to isolate the dependency installation process. This ensures that stages do not re-run unless the dependency files themselves change. The caching mechanism includes: + +- **Dependency Cache**: Both Python and Node.js dependencies are cached in the `/data/olympia/deps` directory. +- **Cache Folders**: Internal cache folders for pip and npm are themselves cached to speed up the build process. + +## GitHub Actions Cache + +The project uses a custom GitHub Actions action (`./.github/actions/cache-deps`) to cache the `/data/olympia/deps` folder. This action significantly increases install times for CI runs by leveraging the GitHub Actions cache. + +```yaml +- name: Cache dependencies + uses: ./.github/actions/cache-deps +``` + +By caching the `/data/olympia/deps` folder, the project ensures that dependencies are quickly restored in CI environments, reducing overall build and test times. + +## Updating/Installing Dependencies + +To update/install all dependencies, run the following command: + +```bash +make up +``` + +This will rebuild the Docker image with the current dependencies specified in the `requirements` and `package.json` files. +We do not support updating dependencies in a running container as the /data/olympia/deps folder is not writable by the olympia user. diff --git a/docs/topics/development/docs.md b/docs/topics/development/docs.md new file mode 100644 index 000000000000..4c666a7d1205 --- /dev/null +++ b/docs/topics/development/docs.md @@ -0,0 +1,18 @@ +# Building Docs + +To simply build the docs: + +```bash +docker compose run web make docs +``` + +If you're working on the docs, use `make loop` to keep your built pages +up-to-date: + +```bash +make shell +cd docs +make loop +``` + +Open `docs/_build/html/index.html` in a web browser. diff --git a/docs/topics/development/docs.rst b/docs/topics/development/docs.rst deleted file mode 100644 index 029c3712b752..000000000000 --- a/docs/topics/development/docs.rst +++ /dev/null @@ -1,16 +0,0 @@ -============= -Building Docs -============= - -To simply build the docs:: - - docker compose run web make docs - -If you're working on the docs, use ``make loop`` to keep your built pages -up-to-date:: - - make shell - cd docs - make loop - -Open ``docs/_build/html/index.html`` in a web browser. diff --git a/docs/topics/development/error_pages.rst b/docs/topics/development/error_pages.md similarity index 55% rename from docs/topics/development/error_pages.rst rename to docs/topics/development/error_pages.md index 14340967c210..c416894dd8b8 100644 --- a/docs/topics/development/error_pages.rst +++ b/docs/topics/development/error_pages.md @@ -1,13 +1,13 @@ -.. _error: +(error)= -=========== -Error Pages -=========== +# Error Pages When running Django locally you get verbose error pages instead of the standard ones. To access the HTML for the standard error pages, you can -access the urls:: +access the urls: - /services/403 - /services/404 - /services/500 +``` +/services/403 +/services/404 +/services/500 +``` diff --git a/docs/topics/development/github_actions.md b/docs/topics/development/github_actions.md new file mode 100644 index 000000000000..a7a772094072 --- /dev/null +++ b/docs/topics/development/github_actions.md @@ -0,0 +1,179 @@ +# Github Actions + +The **addons-server** project uses GitHub Actions to automate testing and building processes in the CI environment. +The CI pipeline broken into workflows, jobs and steps that run on containers in github's cloud environment. +All workflows are defined in the _.github/workflows_ directory. + +## Overview + +Here’s an overview of the existing CI workflows and their architecture: + +Workflows are run on events. Events determine the reason for running the workflow. +A workflow can be triggered by multiple events. {ref}`Learn more ` about configuring events below. + +Generally, we run addons-server inside it's own docker container. This means every workflow generally follows the same pattern. + +### Build the docker image + +Run the {ref}`build-docker ` action to build the docker image from the current ref in github. +This action can be configured to your needs to build, push, and or upload the image to be used later. +The build passing is itself a good first step to test the code, and you can further test after the image is ready. + +### Run the docker image + +Define a job that uses the {ref}`run-docker ` reusable action. This action runs our docker compose +project in the CI container allowing you to exeucte the same commands you would run locally. +This is the foundation of nearly every test we have in CI. + +(configuration)= +## Configuration + +- links to docs for github action configuration and event payloads and syntax +- reusable actions have a _ prefix +- prefer reusable workflows over actions + +### Workflows + +Workflows are the top level configuration object for github actions. Here is an example: + +```yaml +name: Set a name + +on: + pull_request: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}}-${{ toJson(inputs) }} + cancel-in-progress: true + +jobs: + job1: + ... +``` + +> Important: Always define the most specific event triggers possible +> and always set concurrency groups to prevent too many instances of a workflow running simultaneously + +### Reusable Workflows + +We use reusable workflows to enable calling one workflow from another. +This allows better encapsulation of a set of logic that might itself require multiple jobs, matrix jobs +or direct access to built in context like secrets. Otherwise they are conceptually similar to {ref}`reusable actions `. + +Reusable workflows should define a constant in their _concurrency:group_ to prevent deadlock with their triggering workflow. + +```yaml +concurrency: + group: static-${{ github.workflow }}... +``` + +The unique and static key prevents the worfklow (which will match the calling workflow) concurrency group from over matching. + +(_test_yml)= +#### _test.yml + +[link](../../../.github/workflows/_test.yml) + +Our main testing workflow runs a suite of tests verifying the docker image and django code within are running as expected. + +(_test_main_yml_)= +#### _test_main.yml + +[link](../../../.github/workflows/_test_main.yml) + +This workflow is a branch of our _test.yml workflow, running specifically the main pytest suite. +It is split to its own workflow because it runs via a matrix strategy and spins up a lot of jobs. + +(reusable_actions)= +### Reusable Actions + +Reusable actions are like reusable workflows but they do not run on their own runner, +but directly as a step in a given workflow runner container. + +(actions_build_docker)= +#### build-docker + +[link](../../../.github/actions/build-docker/action.yml) + +The main action to build our docker image. +It builds a docker image based on the current state of the code, setting the appropriate metadata +based on context. + +(actions_run_docker)= +#### run-docker + +[link](../../../.github/actions/run-docker/action.yml) + +The main action to run our docker compose project. This action is configurable to run a specified command, with specified services, +and even configurable compose file. Importantly this action will pull an image via the digest or version, and if it cannot find the image +will build it locally to run the current state of the codebase. + +### Actions vs Workflows + +Some of our reusable logic is in reusable actions and some in workflows. There are some important tradeoffs worth mentioning +that inform the decision for which to choose in a particular use case. + +1. Actions run ON a job, workflows run AS a job. If the logic you need requires context from the calling job, +like authentication credentials, created files, etc, then an action is the way to go. Workflows are great for code isolation +or if your logic might benefit itself from splitting to multiple jobs. + +2. Actions are steps. Actions run as a step ON a job (see above) so they checkout code, they cannot access secrets, +they cannot define their own runner or set timeouts or environment variables. Actions should be for very isolated logic +that really executes a single step in a job. + +3. Workflows have their own concurrency considerations. When using reusable workflows the concurrency group +can clash with the current workflow or even (if not careful) with other workfllows. Be careful and set strong concurrency groups. + +4. Workflow jobs are collapsed in the github action UI. This is a nice feature if you need to trigger many jobs in parallel, +like {ref}`test_main <_test_main_yml_>` does. Each of the jobs are collapsible in the UI making it easier to clean up the view. + +For the most part actions are simpler and should be the go to method for extacting reusable logic. Workflows are nice +when you need to organize the logic into multiple jobs or require extreme isolation from the rest of the workflow. + +## Gotchas + +### workflow_dispatch and workflow_call inputs should be identical + +Best practice should be to define all _reusable workflows with both a _workflow_dispatch_ and _workflow_call_ event trigger. +The inputs for each should be identical. This allows testing and otherwise triggering reusable workflows directly or via +another workflow with the same parameters and expectations. + +### github object contents depend on the event triggering the workflow + +One of the reasons we have the context action is because the information embedded in the github +object depends on the event that triggered a workflow, making finding a certain piece of information depend on the 'context'. +Be careful using the github object directly as you must consider many edge cases. Consult the context action and potentially +introduce an important contextual value their so it can be made consistent across different contexts. + +### github converts job/step/workflow outputs to string regardless of the underlying data type + +Even if you define an input with a specific datatype, outputs for steps, jobs and reusable workflows are all converted to strings. +This is important when passing values from outputs to inputs as the data type might not actually be what you want or expect. + +Use: + +```yaml +uses: +with: + number: ${{ fromJSON(output.value) }} +``` + +to convert values back into numbers/booleans. + +### secrets are not available for workflows running on forks. + +Github actions prevents forks from accessing secrets, so workflows that use secrets should be configured to either +not rely on secrets or have fallback behaviour in place. + +### use context action to define global context + +Most global context should be defined in the context action instead of directly in workflows. + +### prevent invalid workflow configurations + +When reusable workflows are passed invalid outputs, github will silently fail the workflow, to prevent this you should always +check the outcome of reusable workflow calls. + +[gar_link]: https://cloud.google.com/artifact-registry diff --git a/docs/topics/development/index.md b/docs/topics/development/index.md new file mode 100644 index 000000000000..13bc9ec9f1ab --- /dev/null +++ b/docs/topics/development/index.md @@ -0,0 +1,33 @@ +# Development + +Welcome to the development section of the **addons-server** documentation. This guide is designed to help developers quickly understand, set up, and manage the project's various services. By leveraging Docker Compose, we ensure modularity, isolation, and efficiency across development and testing environments. Here’s a concise overview of each section to get you up and running fast. + +```{toctree} +:maxdepth: 2 + +setup_and_configuration +building_and_running_services +makefile_commands +testing_and_quality_assurance +data_management +dependency_management +performance_and_optimization +localization_and_internationalization +troubleshooting_and_debugging +github_actions +error_pages +style +contributing +branching +vpn +acl +logging +static-files +search +docs +waffle +remote_settings +../../../README.rst +authentication +scanner_pipeline +``` diff --git a/docs/topics/development/index.rst b/docs/topics/development/index.rst deleted file mode 100644 index c67c7e70726e..000000000000 --- a/docs/topics/development/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -=========== -Development -=========== - -.. toctree:: - :maxdepth: 2 - - tests - debugging - dependencies - error_pages - testing - style - contributing - branching - vpn - acl - logging - translations - search - docs - waffle - ../../../README.rst diff --git a/docs/topics/development/localization_and_internationalization.md b/docs/topics/development/localization_and_internationalization.md new file mode 100644 index 000000000000..e53db67b719e --- /dev/null +++ b/docs/topics/development/localization_and_internationalization.md @@ -0,0 +1,111 @@ +# Localization and Internationalization + +Localization and internationalization are important aspects of the **addons-server** project, ensuring that the application can support multiple languages and locales. This section covers the key concepts and processes for managing localization and internationalization. + +## Locale Management + +Locale management involves compiling and managing translation files. The **addons-server** project uses a structured approach to handle localization files efficiently. + +1. **Compiling Locales**: + - The Makefile provides commands to compile locale files, ensuring that translations are up-to-date. + - Use the following command to compile locales: + + ```sh + make compile_locales + ``` + +2. **Managing Locale Files**: + - Locale files are typically stored in the `locale` directory within the project. + - The project structure ensures that all locale files are organized and easily accessible for updates and maintenance. + +## Adding New Translations + +We write the english translations of our strings directly in the code, using the `gettext` function. For example: + +```python +from django.utils.translation import gettext_lazy as _ + +def my_view(request): + output = _('Welcome to my site.') + return HttpResponse(output) +``` + +When developing locally you should not really need to do anything special to see the translations. The `gettext` function will return the string as is if it can't find a translation for it. In CI translation strings are automatically extracted and uploaded to Pontoon for translation. + +## Translation Management + +Translation management involves handling translation strings and merging them as needed. The **addons-server** project follows best practices to ensure that translations are accurate and consistent. + +1. **Handling Translation Strings**: + - Translation strings are extracted from the source code and stored in `.po` files. + - The `.po` file format is used to manage locale strings, providing a standard way to handle translations. + +2. **Merging Translation Strings**: + - To extract new locales from the codebase, use the following command: + + ```sh + make extract_locales + ``` + + - This command scans the codebase and updates the `.po` files with new or changed translation strings. + - After extraction, scripts are used to merge new or updated translation strings into the existing locale files. + - This process ensures that all translations are properly integrated and maintained. + +## Additional Tools and Practices + +1. **Pontoon**: + - The **addons-server** project uses Pontoon, Mozilla's localization service, to manage translations. + - Pontoon provides an interface for translators to contribute translations and review changes, ensuring high-quality localization. + +2. **.po File Format**: + - The `.po` file format is a widely used standard for managing translation strings. + - It allows for easy editing and updating of translations, facilitating collaboration among translators. + +## Translating Fields on Models + +The `olympia.translations` app defines a `olympia.translations.models.Translation` model, but for the most part, you shouldn't have to use that directly. When you want to create a foreign key to the `translations` table, use `olympia.translations.fields.TranslatedField`. This subclasses Django's `django.db.models.ForeignKey` to make it work with our special handling of translation rows. + +### Minimal Model Example + +A minimal model with translations in addons-server would look like this: + +```python +from django.db import models + +from olympia.amo.models import ModelBase +from olympia.translations.fields import TranslatedField, save_signal + +class MyModel(ModelBase): + description = TranslatedField() + +models.signals.pre_save.connect(save_signal, + sender=MyModel, + dispatch_uid='mymodel_translations') +``` + +### How It Works Behind the Scenes + +A `TranslatedField` is actually a `ForeignKey` to the `translations` table. To support multiple languages, we use a special feature of MySQL allowing a `ForeignKey` to point to multiple rows. + +#### When Querying + +Our base manager has a `_with_translations()` method that is automatically called when you instantiate a queryset. It does two things: + +- Adds an extra `lang=lang` in the query to prevent query caching from returning objects in the wrong language. +- Calls `olympia.translations.transformers.get_trans()` which builds a custom SQL query to fetch translations in the current language and fallback language. + +This custom query ensures that only the specified languages are considered and uses a double join with `IF`/`ELSE` for each field. The results are fetched using a slave database connection to improve performance. + +#### When Setting + +Every time you set a translated field to a string value, the `TranslationDescriptor` `__set__` method is called. It determines whether it's a new translation or an update to an existing translation and updates the relevant `Translation` objects accordingly. These objects are queued for saving, which happens on the `pre_save` signal to avoid foreign key constraint errors. + +#### When Deleting + +Deleting all translations for a field is done using `olympia.translations.models.delete_translation()`, which sets the field to `NULL` and deletes all attached translations. Deleting a specific translation is possible but not recommended due to potential issues with fallback languages and foreign key constraints. + +### Ordering by a Translated Field + +`olympia.translations.query.order_by_translation` allows you to order a `QuerySet` by a translated field, honoring the current and fallback locales like when querying. + +By following these practices, the **addons-server** project ensures that the application can support multiple languages and locales effectively. For more detailed instructions, refer to the project's Makefile and locale management scripts in the repository. diff --git a/docs/topics/development/logging.md b/docs/topics/development/logging.md new file mode 100644 index 000000000000..20a496145dcc --- /dev/null +++ b/docs/topics/development/logging.md @@ -0,0 +1,77 @@ +(logging)= + +# Logging + +Logging is fun. We all want to be lumberjacks. My muscle-memory wants to put +`print` statements everywhere, but it's better to use `log.debug` instead. +`print` statements make mod_wsgi sad, and they're not much use in production. +Plus, `django-debug-toolbar` can hijack the logger and show all the log +statements generated during the last request. When `DEBUG = True`, all logs +will be printed to the development console where you started the server. In +production, we're piping everything into `mozlog`. + +## Configuration + +The root logger is set up from `settings_base` in the `src/olympia/lib` +of addons-server. It sets up sensible defaults, but you can tweak them to your liking: + +### Log level + +There is no unified log level, instead every logger has it's own log level +because it depends on the context they're used in. + +### LOGGING + +See PEP 391 for formatting help. Messages will not propagate through a +logger unless `propagate: True` is set. + +> ```python +> LOGGING = { +> 'loggers': { +> 'caching': {'handlers': ['null']}, +> }, +> } +> ``` + +If you want to add more to this do something like this: + +``` +LOGGING['loggers'].update({ + 'z.paypal': { + 'level': logging.DEBUG, + }, + 'z.es': { + 'handlers': ['null'], + }, +}) +``` + +## Using Loggers + +The `olympia.core.logger` package uses global objects to make the same +logging configuration available to all code loaded in the interpreter. Loggers +are created in a pseudo-namespace structure, so app-level loggers can inherit +settings from a root logger. olympia's root namespace is just `"z"`, in the +interest of brevity. In the caching package, we create a logger that inherits +the configuration by naming it `"z.caching"`: + +```python +import olympia.core.logger + +log = olympia.core.logger.getLogger('z.caching') + +log.debug("I'm in the caching package.") +``` + +Logs can be nested as much as you want. Maintaining log namespaces is useful +because we can turn up the logging output for a particular section of olympia +without becoming overwhelmed with logging from all other parts. + +### olympia.core.logging vs. logging + +`olympia.core.logger.getLogger` should be used everywhere. It returns a +`LoggingAdapter` that inserts the current user's IP address and username into +the log message. For code that lives outside the request-response cycle, it +will insert empty values, keeping the message formatting the same. + +Complete logging docs: diff --git a/docs/topics/development/logging.rst b/docs/topics/development/logging.rst deleted file mode 100644 index e47eebfa1729..000000000000 --- a/docs/topics/development/logging.rst +++ /dev/null @@ -1,81 +0,0 @@ -.. _logging: - -======= -Logging -======= - -Logging is fun. We all want to be lumberjacks. My muscle-memory wants to put -``print`` statements everywhere, but it's better to use ``log.debug`` instead. -``print`` statements make mod_wsgi sad, and they're not much use in production. -Plus, ``django-debug-toolbar`` can hijack the logger and show all the log -statements generated during the last request. When ``DEBUG = True``, all logs -will be printed to the development console where you started the server. In -production, we're piping everything into ``mozlog``. - - -Configuration -------------- - -The root logger is set up from ``settings_base`` in the ``src/olympia/lib`` -of addons-server. It sets up sensible defaults, but you can tweak them to your liking: - -Log level -~~~~~~~~~ -There is no unified log level, instead every logger has it's own log level -because it depends on the context they're used in. - -LOGGING -~~~~~~~ -See PEP 391 for formatting help. Messages will not propagate through a -logger unless ``propagate: True`` is set. - - :: - - LOGGING = { - 'loggers': { - 'caching': {'handlers': ['null']}, - }, - } - -If you want to add more to this do something like this:: - - LOGGING['loggers'].update({ - 'z.paypal': { - 'level': logging.DEBUG, - }, - 'z.es': { - 'handlers': ['null'], - }, - }) - - -Using Loggers -------------- - -The ``olympia.core.logger`` package uses global objects to make the same -logging configuration available to all code loaded in the interpreter. Loggers -are created in a pseudo-namespace structure, so app-level loggers can inherit -settings from a root logger. olympia's root namespace is just ``"z"``, in the -interest of brevity. In the caching package, we create a logger that inherits -the configuration by naming it ``"z.caching"``:: - - import olympia.core.logger - - log = olympia.core.logger.getLogger('z.caching') - - log.debug("I'm in the caching package.") - -Logs can be nested as much as you want. Maintaining log namespaces is useful -because we can turn up the logging output for a particular section of olympia -without becoming overwhelmed with logging from all other parts. - - -olympia.core.logging vs. logging -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``olympia.core.logger.getLogger`` should be used everywhere. It returns a -``LoggingAdapter`` that inserts the current user's IP address and username into -the log message. For code that lives outside the request-response cycle, it -will insert empty values, keeping the message formatting the same. - -Complete logging docs: http://docs.python.org/library/logging.html diff --git a/docs/topics/development/makefile_commands.md b/docs/topics/development/makefile_commands.md new file mode 100644 index 000000000000..4b5a03e3be95 --- /dev/null +++ b/docs/topics/development/makefile_commands.md @@ -0,0 +1,129 @@ +# Makefile Commands + +The Makefile for the **addons-server** project provides a convenient interface for interacting with the Docker environment and managing common development tasks. This section details the key commands and their purposes. + +## Overview of Makefile + +The Makefile automates various tasks, reducing the need for manual Docker and shell commands. This ensures consistency and streamlines development workflows. + +## Makefile Structure + +The **addons-server** project splits its Makefile configuration into three files to separate concerns between the host operating system and Docker container environments: + +1. **Makefile**: + - Acts as the main entry point and delegates commands to either `Makefile-os` or `Makefile-docker` based on the environment. + +2. **Makefile-os**: + - Contains targets designed to run on the host operating system. + - **Gotcha**: If you run a command specified in `Makefile-os` inside the container (e.g., by running `make shell` and then the command), it might not be available because Make will ignore those commands. + +3. **Makefile-docker**: + - Contains targets designed to run inside the Docker container. + - If you run a target defined in `Makefile-docker` from the host, Make will redirect the command to the running container by prefixing the relevant `docker-compose exec` command. + +A common benefit of using Makefiles in this manner is the ability to coordinate complex behaviors that work in both local and CI environments from a single place. It also helps organize commands meant to be run on the host machine or inside a running container. + +**Note**: We aim to keep the majority of logic defined within the Makefiles themselves. However, if the logic becomes overly complex, it can be defined in a `./scripts/*` file and executed via a Make command. + +## Common Commands + +1. **`setup`**: + - **Purpose**: Initializes the project by creating necessary configuration files, including the `.env` file. + - **Usage**: + + ```sh + make setup + ``` + +2. **`up`**: + - **Purpose**: Builds the Docker image using BuildKit and Bake, and starts the containers as defined in the Docker Compose configuration. + - **Usage**: + + ```sh + make up + ``` + +3. **`down`**: + - **Purpose**: Stops and removes the running containers. + - **Usage**: + + ```sh + make down + ``` + +4. **`djshell`**: + - **Purpose**: Provides access to the Django shell within the `web` container. + - **Usage**: + + ```sh + make djshell + ``` + +5. **`shell`**: + - **Purpose**: Provides access to a shell within the `web` container. + - **Usage**: + + ```sh + make shell + ``` + +6. **`test`**: + - **Purpose**: Executes the entire test suite using pytest. + - **Usage**: + + ```sh + make test + ``` + +7. **`lint`**: + - **Purpose**: Enforces code style and quality standards using various linters. + - **Usage**: + + ```sh + make lint + ``` + +8. **`format`**: + - **Purpose**: Automatically formats the codebase according to predefined style guidelines. + - **Usage**: + + ```sh + make format + ``` + +## Specialized Commands + +1. **`data_export` and `data_restore`**: + - **Purpose**: Facilitates exporting and restoring data from the MySQL database. + - **Usage**: + + ```sh + make data_export + make data_restore + ``` + +2. **`build_docker_image`**: + - **Purpose**: Builds the Docker image using BuildKit and Bake. + - **Usage**: + + ```sh + make build_docker_image + ``` + +## Forcing a Specific Makefile + +You can force Make to run a specific command from a particular Makefile by specifying the file: + +```sh +make -f +``` + +## Running Commands Inside the Container + +If you run a target defined in `Makefile-docker` from the host, Make will redirect the command to the running container. If the containers are not running, this might fail, and you will need to ensure the containers are running by executing: + +```sh +make up +``` + +By using these Makefile commands, developers can streamline their workflow, ensuring consistency and efficiency in their development process. For more detailed information on each command, refer to the comments and definitions within the Makefiles themselves. diff --git a/docs/topics/development/performance_and_optimization.md b/docs/topics/development/performance_and_optimization.md new file mode 100644 index 000000000000..6f812536973d --- /dev/null +++ b/docs/topics/development/performance_and_optimization.md @@ -0,0 +1,52 @@ +# Performance and Optimization + +Optimizing performance is essential for maintaining efficient development and deployment workflows. This section covers the key strategies and tools used in the **addons-server** project for performance and optimization. + +## Docker Layer Caching + +Docker layer caching is a powerful feature that significantly speeds up the build process by reusing unchanged layers. This section explains the benefits and setup for Docker layer caching in the **addons-server** project. + +1. **Benefits of Docker Layer Caching**: + - **Reduced Build Times**: By caching intermediate layers, Docker can reuse these layers in subsequent builds, reducing the overall build time. + - **Efficient Resource Usage**: Caching helps save bandwidth and computational resources by avoiding redundant downloads and computations. + - **Consistency**: Ensures that identical builds produce identical layers, promoting consistency across builds. + +2. **Setup for Docker Layer Caching**: + - **Build Stages**: The Dockerfile uses build stages to isolate dependency installation and other tasks. This ensures that stages are only re-executed when necessary. + - **Cache Mounts**: The project uses `--mount=type=cache` in the Dockerfile to cache directories across builds. This is particularly useful for caching Python and npm dependencies, speeding up future builds. + + Example snippet from the Dockerfile: + + ```Dockerfile + RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements/prod.txt + RUN --mount=type=cache,target=/root/.npm npm install + ``` + + - **BuildKit**: Ensures BuildKit is enabled to take advantage of advanced caching features: + + ```sh + export DOCKER_BUILDKIT=1 + ``` + + - **GitHub Actions Cache**: The custom action (`./.github/actions/cache-deps`) caches the `/data/olympia/deps` folder, leveraging GitHub Actions cache to improve CI run times. + +## Performance Testing + +Performance testing is crucial for identifying bottlenecks and optimizing application performance. The **addons-server** project includes various strategies for performance testing and optimization. + +1. **Running Performance Tests**: + - The project uses `pytest` along with plugins like `pytest-split` and `pytest-xdist` to run tests in parallel, significantly reducing test times. + - Performance-specific tests can be run to measure the responsiveness and efficiency of the application. + +2. **Optimization Tips**: + - **Parallel Testing**: Use `pytest-xdist` to run tests in parallel: + + ```sh + pytest -n auto + ``` + + - **Test Splitting**: Use `pytest-split` to distribute tests evenly across multiple processes. + - **Code Profiling**: Use profiling tools to identify slow functions and optimize them. + - **Database Optimization**: Regularly monitor and optimize database queries to ensure efficient data retrieval and storage. + +By implementing these performance and optimization strategies, the **addons-server** project ensures efficient and reliable builds and tests, both locally and in CI environments. For more detailed instructions, refer to the project's Dockerfile, Makefile, and GitHub Actions configurations in the repository. diff --git a/docs/topics/development/remote_settings.md b/docs/topics/development/remote_settings.md new file mode 100644 index 000000000000..367b2d5bff5a --- /dev/null +++ b/docs/topics/development/remote_settings.md @@ -0,0 +1,116 @@ +# Remote Settings + +This page explains how to set up [Remote Settings][] locally, which can be +useful when working on the [Blocklist feature](../blocklist.md). AMO must be +running locally first. Make sure it's available at . + +## Configure addons-server + +Add the following configuration variables to the `local_settings.py` file: + +``` +# When using Docker Desktop - `host.docker.internal` is a special host to allow +# containers to get access to the host system, but that won't work on Linux. +REMOTE_SETTINGS_API_URL = "http://host.docker.internal:8888/v1/" +REMOTE_SETTINGS_WRITER_URL = "http://host.docker.internal:8888/v1/" + +# For Linux, you need to find the IP address of the Remote Settings container: +# REMOTE_SETTINGS_API_URL = "http://172.17.0.1:8888/v1/" +# REMOTE_SETTINGS_WRITER_URL = "http://172.17.0.1:8888/v1/" +``` + +Next, reload everything by running `make up`. + +At this point, AMO should be able to find the Remote Settings local server that +we're going to set up next. + +## Set up Remote Settings + +In order to set up Remote Settings, follow these steps: + +1. Clone +2. Run `make start` in the `remote-settings` repository +3. Add `127.0.0.1 autograph` to your `/etc/hosts` file + +Verify that Remote Settings is healthy: + +``` +curl http://127.0.0.1:8888/v1/__heartbeat__ +{ + "storage": true, + "permission": true, + "cache": true, + "attachments": true, + "signer": true +} +``` + +## Configure the user/permissions + +First, we need an `admin` account. We can create one with the Remote Settings +API: + +``` +curl -X PUT -H 'Content-Type: application/json' \ + -d '{"data": {"password": "s3cr3t"}}' \ + http://127.0.0.1:8888/v1/accounts/admin +``` + +Next, we need a user for AMO: + +``` +curl -X PUT -H 'Content-Type: application/json' \ + -d '{"data": {"password": "amo_remote_settings_password"}}' \ + http://127.0.0.1:8888/v1/accounts/amo_remote_settings_username +``` + +We then need to give this user _write_ access to the `staging` bucket so that it +can create the `addons-bloomfilters` collection. This is where AMO will write +the new records, which will be propagated to the public bucket/collection +automatically: + +``` +curl -X PUT -H 'Content-Type: application/json' \ + -d '{"permissions": {"write": ["account:amo_remote_settings_username"]}}' \ + -u admin:s3cr3t \ + http://127.0.0.1:8888/v1/buckets/staging +``` + +``` +curl -X PUT -H 'Content-Type: application/json' \ + -u amo_remote_settings_username:amo_remote_settings_password \ + http://127.0.0.1:8888/v1/buckets/staging/collections/addons-bloomfilters +``` + +At this point, AMO should be able to authenticate to Remote Settings. This can +be verified with the following command: + +``` +curl http://olympia.test/services/monitor.json +{ + "cinder": { + "state": true, + "status": "" + }, + "rabbitmq": { + "state": true, + "status": "" + }, + "remotesettings": { + "state": true, + "status": "" + }, + "signer": { + "state": true, + "status": "" + } +} +``` + +After AMO uploads records, the Remote Settings `addons-bloomfilters` collection +will be available at: + + +We are done \o/ + +[Remote Settings]: https://remote-settings.readthedocs.io/en/latest/index.html diff --git a/docs/topics/development/scanner_pipeline.md b/docs/topics/development/scanner_pipeline.md new file mode 100644 index 000000000000..d3f75361ee09 --- /dev/null +++ b/docs/topics/development/scanner_pipeline.md @@ -0,0 +1,99 @@ +# Scanner Pipeline + +The scanner pipeline is a security feature to run scanners. + +## Scanner Webhooks + +A scanner webhook is essentially a URL to a service subscribed to events that +occur on AMO. These webhooks are registed in the AMO (django) admin. + +When a [scanner webhook event](#scanner-webhook-events) occurs, AMO will send an +HTTP request to each webhook subscribed to this event. The payload sent to the +webook depends on the event. The response from each webhook will lead to the +creation of a [scanner result](#scanner-results). + +Each service registered as a scanner webhook must be protected with a secret +(bearer) token. AMO will include this token in the `Authorization` header of +each HTTP request made to the webhook. + +### Adding a scanner webhook + +Scanner webhooks must be registered in the AMO (django) admin. The following +information must be provided: + +- _Name_: the name of the scanner +- _URL_: the full URL of the scanner, which will receive the events +- _API key_: the secret key sent to the scanner to authenticate AMO requests + +Add one or more scanner webhook events, see the next section for more +information. + +(scanner-webhook-events)= +### Scanner Webhook Events + +#### `during_validation` + +This event occurs when a file upload is being validated, which typically happens +when a new version is being submitted to AMO. This is called near the end of the +validation chain. + +The payload sent looks like this. Assuming correct permissions, the URL in +`download_url` allows the services notified for this event to download the (raw) +uploaded file. + +```json +{ + "download_url": "http://olympia.test/uploads/file/42" +} +``` + +#### `on_source_code_uploaded` + +This event occurs when source code is uploaded, e.g., in DevHub. + +The payload sent looks like this: + +```json +{ + "addon_id": 85, + "version_id": 42, + "download_source_url": "http://olympia.test/downloads/source/42", + "license_slug": "MPL-2.0", + "activity_log_id": 2170 +} +``` + +### Adding a new event + +1. Add a constant for the new event in `src/olympia/constants/scanners.py`. The + name must start with `WEBHOOK_`. Make sure the new constant is registered in + `WEBHOOK_EVENTS` (in the same file). +2. In a `tasks.py` file, create a Celery task that calls `call_webhooks(event_name, +payload, upload=none, version=None)`. Make sure this task is assigned to a + queue in `src/olympia/lib/settings_base.py`. +3. Invoke this Celery task (with `.delay()`) where the event occurs in the code. +4. Update this documentation page. + +(scanner-results)= +## Scanner Results + +A scanner result stores the output returned by a scanner, and it might be tied +to a [webhook event](#scanner-webhook-events). + +Scanners can return a list of _matched rules_. When these rules exist as +[scanner rules](#scanner-rules) on AMO, it becomes possible to execute [scanner +actions](#scanner-actions). This is a core concept of the scanner pipeline, +which essentially allows a scanner to make a change to an add-on version. + +(scanner-rules)= +## Scanner Rules + +A scanner rule allows scanners to trigger an [action](#scanner-actions). + +(scanner-actions)= +## Scanner Actions + +A scanner action is some logic that can be applied to an add-on version. For +example, flagging a version for manual review is a scanner action. + +These actions are defined in `src/olympia/scanners/actions.py`. diff --git a/docs/topics/development/search.md b/docs/topics/development/search.md new file mode 100644 index 000000000000..05c53d13b48d --- /dev/null +++ b/docs/topics/development/search.md @@ -0,0 +1,114 @@ +(amo-search-explainer)= + +# How does search on AMO work? + +## High-level overview + +AMO add-ons are indexed in our Elasticsearch cluster. For each search query +someone makes on AMO, we run a custom set of full-text queries against that +cluster. + +Our autocomplete (that you can see when starting to type a few characters in +the search field) uses the exact same implementation as a regular search +underneath. + +### Rules + +For each search query, we apply a number of rules that attempt to find the +search terms in each add-on name, summary and description. Each rule generates +a score that depends on: + +> - The frequency of the terms in the field we're looking at +> - The importance of each term in the overall index (the more common the term is across all add-ons, the less it impacts the score) +> - The length of the field (shorter fields give a higher score as the search term is considered more relevant if they make up a larger part of the field) + +Each rule is also given a specific boost affecting its score, making matches +against the add-on name more important and matches against the summary or +description. + +Add-on names receive special treatment: Partial or misspelled matches are +accepted to some extent while exact matches receive a significantly higher +score. + +### Scoring + +Each score for each rule is combined into a final score which we modify +depending on the add-on popularity on a logarithm scale. "Recommended" and +"By Firefox" add-ons get an additional, significant boost to their score. + +Finally, results are returned according to their score in descending order. + +## Technical overview + +We store two kinds of data in the _addons_ index: indexed fields that are used for search purposes, and non-indexed fields that are meant to be returned (often as-is with no transformations) by the search API (allowing us to return search results data without hitting the database). The latter is not relevant to this document. + +Our search can be reached either via the API through {ref}`/api/v5/addons/search/ ` or {ref}`/api/v5/addons/autocomplete/ ` which are used by our frontend. + +### Indexing + +The key fields we search against are `name`, `summary` and `description`. Because all can be translated, we index them multiple times: + +> - Once with the translation in the default locale of the add-on, under `{field}`, analyzed with just the `snowball` analyzer for `description` and `summary`, and a custom analyzer for `name` that applies the following filters: `standard`, `word_delimiter` (a custom version with `preserve_original` set to `true`), `lowercase`, `stop`, and `dictionary_decompounder` (with a specific word list) and `unique`. +> - Once for every translation that exists for that field, using Elasticsearch language-specific analyzer if supported, under `{field}_l10n_{analyzer}`. + +**In addition, for the name, we also have:** + +- For all fields described above also contains a subfield called `raw` that holds a non-analyzed variant for exact matches in the corresponding language (stored as a `keyword`, with a `lowercase` normalizer). +- A `name.trigram` variant for the field in the default language, which is using a custom analyzer that depends on a `ngram` tokenizer (with `min_gram=3`, `max_gram=3` and `token_chars=["letter", "digit"]`). + +### Flow of a search query through AMO + +Let's assume we search on addons-frontend (not legacy) the search query hits the API and gets handled by `AddonSearchView`, which directly queries ElasticSearch and doesn't involve the database at all. + +There are a few filters that are described in the {ref}`/api/v5/addons/search/ docs ` but most of them are not very relevant for text search queries. Examples are filters by guid, platform, category, add-on type or appversion (application version compatibility). Those filters are applied using a `filter` clause and shouldn't affect scoring. + +Much more relevant for text searches (and this is primarily used when you use the search on the frontend) is `SearchQueryFilter`. + +It composes various rules to define a more or less usable ranking: + +#### Primary rules + +These are the ones using the strongest boosts, so they are only applied to the add-on name. + +**Applied rules** (merged via `should`): + +1. A `dis_max` query with `term` matches on `name_l10n_{analyzer}.raw` and `name.raw` if the language of the request matches a known language-specific analyzer, or just a `term` query on `name.raw` (`boost=100.0`) otherwise - our attempt to implement exact matches +2. If we have a matching language-specific analyzer, we add a `match` query to `name_l10n_{analyzer}` (`boost=5.0`, `operator=and`) +3. A `phrase` match on `name` that allows swapped terms (`boost=8.0`, `slop=1`) +4. A `match` on `name`, using the standard text analyzer (`boost=6.0`, `analyzer=standard`, `operator=and`) +5. A `prefix` match on `name` (`boost=3.0`) +6. If a query is \< 20 characters long, a `dis_max` query (`boost=4.0`) composed of a fuzzy match on `name` (`boost=4.0`, `prefix_length=2`, `fuzziness=AUTO`, `minimum_should_match=2<2 3<-25%`) and a `match` query on `name.trigram`, with a `minimum_should_match=66%` to avoid noise + +#### Secondary rules + +These are the ones using the weakest boosts, they are applied to fields containing more text like description, summary and tags. + +**Applied rules** (merged via `should`): + +1. Look for matches inside the summary (`boost=3.0`, `operator=and`) +2. Look for matches inside the description (`boost=2.0`, `operator=and`) + +If the language of the request matches a known language-specific analyzer, those are made using a `multi_match` query using `summary` or `description` and the corresponding `{field}_l10n_{analyzer}`, similar to how exact name matches are performed above, in order to support potential translations. + +#### Scoring + +We combine scores through a `function_score` query that multiplies the score by several factors: + +> - A first multiplier is always applied through the `field_value_factor` function on `average_daily_users` with a `log2p` modifier +> - An additional `4.0` weight is applied if the add-on is public & non-experimental. +> - Finally, `5.0` weight is applied to By Firefox and Recommended add-ons. + +On top of the two sets of rules above, a `rescore` query is applied with a `window_size` of `10`. In production, we have 5 shards, so that should re-adjust the score of the top 50 results returned only. The rules used for rescoring are the same used in the secondary rules above, with just one difference: it's using `match_phrase` instead of `match`, with a slop of `10`. + +#### General query flow + +> 1. Fetch current translation +> 2. Fetch locale specific analyzer ([List of analyzers](https://github.com/mozilla/addons-server/blob/f099b20fa0f27989009082c1f58da0f1d0a341a3/src/olympia/constants/search.py#L13-L52)) +> 3. Apply primary and secondary *should* rules +> 4. Determine the score +> 5. Rescore the top 10 results per shard + +#### See also + +> - [addons-server search ranking tests](https://github.com/mozilla/addons-server/blob/master/src/olympia/search/tests/test_search_ranking.py) +> - [Elasticsearch relevancy algorithm](https://www.elastic.co/blog/practical-bm25-part-2-the-bm25-algorithm-and-its-variables) diff --git a/docs/topics/development/search.rst b/docs/topics/development/search.rst deleted file mode 100644 index 61f53731e1cf..000000000000 --- a/docs/topics/development/search.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. _amo_search_explainer: - -============================ -How does search on AMO work? -============================ - -.. note:: - - This is documenting our current state of how search is implemented in addons-server. - We will be using this to plan future improvements so please note that we are - aware that the process written below is not perfect and has bugs here and there. - - Please see https://github.com/orgs/mozilla/projects/17#card-10287357 for more planning. - - -General structure -================= - -Our Elasticsearch cluster contains Add-ons (``addons`` index) and statistics data. The purpose of that document is to describe the add-ons part only though. We store two kinds of data for add-ons: indexed fields that are used for search purposes, and non-indexed fields that are meant to be returned (often as-is with no transformations) by the search API (allowing us to return search results data without hitting the database). The latter is not relevant to this document. - -Our search can be reached either via the API through :ref:`/api/v4/addons/search/ ` or :ref:`/api/v4/addons/autocomplete/ ` which are used by our addons-frontend as well as via our legacy pages (which are going away and off-topic here). - - -Indexing -======== - -The key fields we search against are ``name``, ``summary`` and ``description``. Because all can be translated, we index twice: -- Once with the translation in the language-specific analyzer if supported, under ``{field}_l10n_{analyzer}`` -- Once with the translation in the default locale of the add-on, under ``{field}``, analyzed with just the ``snowball`` analyzer for ``description`` and ``summary``, and a custom analyzer for ``name`` that applies the following filters: ``standard``, ``word_delimiter`` (a custom version with ``preserve_original`` set to ``true``), ``lowercase``, ``stop``, and ``dictionary_decompounder`` (with a specific word list) and ``unique``. - -In addition, for the name, both fields also contains a subfield called ``raw`` that holds a non-analyzed variant for exact matches in the corresponding language (stored as a ``keyword``, with a ``lowercase`` normalizer). We also have a ``name.trigram`` variant for the field in the default language, which is using a custom analyzer that depends on a ``ngram`` tokenizer (with ``min_gram=3``, ``max_gram=3`` and ``token_chars=["letter", "digit"]``) - - -Flow of a search query through AMO -================================== - -Let's assume we search on addons-frontend (not legacy) the search query hits the API and gets handled by ``AddonSearchView``, which directly queries ElasticSearch and doesn't involve the database at all. - -There are a few filters that are described in the :ref:`/api/v4/addons/search/ docs ` but most of them are not very relevant for raw search queries. Examples are filters by guid, platform, category, add-on type or appversion (application version compatibility). Those filters are applied using a ``filter`` clause and shouldn't affect scoring. - -Much more relevant for raw add-on searches (and this is primarily used when you use the search on the frontend) is ``SearchQueryFilter``. - -It composes various rules to define a more or less usable ranking: - -Primary rules -------------- - -These are the ones using the strongest boosts, so they are only applied to the add-on name. - -**Applied rules** (merged via ``should``): - -1. A ``dis_max`` query with ``term`` matches on ``name_l10n_{analyzer}.raw`` and ``name.raw`` if the language of the request matches a known language-specific analyzer, or just a ``term`` query on ``name.raw`` (``boost=100.0``) otherwise - our attempt to implement exact matches -2. If we have a matching language-specific analyzer, we add a ``match`` query to ``name_l10n_{analyzer}`` (``boost=5.0``, ``operator=and``) -3. A ``phrase`` match on ``name`` that allows swapped terms (``boost=8.0``, ``slop=1``) -4. A ``match`` on ``name``, using the standard text analyzer (``boost=6.0``, ``analyzer=standard``, ``operator=and``) -5. A ``prefix`` match on ``name`` (``boost=3.0``) -6. If a query is < 20 characters long, a ``dis_max`` query (``boost=4.0``) composed of a fuzzy match on ``name`` (``boost=4.0``, ``prefix_length=2``, ``fuzziness=AUTO``, ``minimum_should_match=2<2 3<-25%``) and a ``match`` query on ``name.trigram``, with a ``minimum_should_match=66%`` to avoid noise. - - -Secondary rules ---------------- - -These are the ones using the weakest boosts, they are applied to fields containing more text like description, summary and tags. - -**Applied rules** (merged via ``should``): - -1. Look for matches inside the summary (``boost=3.0``, ``operator=and``) -2. Look for matches inside the description (``boost=2.0``, ``operator=and``) - -If the language of the request matches a known language-specific analyzer, those are made using a ``multi_match`` query using ``summary`` or ``description`` and the corresponding ``{field}_l10n_{analyzer}``, similar to how exact name matches are performed above, in order to support potential translations. - - -Rescoring rules ---------------- - -On top of the two sets of rules above, a ``rescore`` query is applied with a ``window_size`` of ``10``. In production, we have 5 shards, so that -should re-adjust the score of the top 50 results returned only. The rules used for rescoring are the same used in the secondary rules above, with just one difference: it's using ``match_phrase`` instead of ``match``, with a slop of ``10``. - - -General query flow: -------------------- - - 1. Fetch current translation - 2. Fetch locale specific analyzer (`List of analyzers `_) - 3. Merge primary and secondary *should* rules - 4. Create a ``function_score`` query that uses a ``field_value_factor`` function on ``average_daily_users`` with a ``log2p`` modifier, as well as a ``4.0`` weight if the add-on is public & non-experimental. - 5. Add the ``rescore`` query to the mix diff --git a/docs/topics/development/setup_and_configuration.md b/docs/topics/development/setup_and_configuration.md new file mode 100644 index 000000000000..22b312e143be --- /dev/null +++ b/docs/topics/development/setup_and_configuration.md @@ -0,0 +1,428 @@ +# Setup and Configuration + +This section covers how to run _addons-server_ locally. See [github actions](./github_actions.md) for running in CI. +This should be where you start if you are running _addons-server_ for the first time. +Setting up the local development environment for **addons-server** involves configuring Docker Compose to run the necessary services. +Follow these steps to get started: + +## Prerequisites + +- Ensure Docker and Docker Compose are installed on your system. +- Clone the **addons-server** repository from GitHub: + + ```sh + git clone https://github.com/mozilla/addons-server + cd addons-server + ``` + +## Running the docker compose project + +> TLDR; Just run `make up`. + +The _make up_ command ensures all necessary files are created on the host and starts the Docker Compose project, +including volumes, containers, networks, databases and indexes. +It is meant to be run frequently whenever you want to bring your environment "up". + +Here's a high-level overview of what _make up_ does: + +```make +.PHONY: up +up: setup docker_pull_or_build docker_compose_up docker_clean_images docker_clean_volumes data +``` + +- **setup**: Creates configuration files such as `.env` and `version.json`. +- **docker_pull_or_build**: Pulls or builds the Docker image based on the image version. +- **docker_compose_up**: Starts the Docker containers defined in [docker-compose.yml][docker-compose]. +- **docker_clean_images** and **docker_clean_volumes**: Cleans up unused Docker images and volumes. +- **data**: Ensures the database, seed, and index are created. + +What happens if you run `make up` when your environment is already running?. +Well that depends on what is changed since the last time you ran it. +Because `make up` is {ref}`idempotent ` it will only run the commands that are necessary to bring your environment up to date. +If nothing has changed, nothing will happen because your environment is already in the desired state. + +## Shutting down your environment + +> TLDR; just run `make down` + +The `make down` command does almost the complete opposite of `make up`. +It stops all docker services and removes locally built images and any used volumes. + +Running `make down` will free up resources on your machine and can help if your environment gets stuck in a difficult to debug state. + +A common solution to many problems is to run `make down && make up`. + +> NOTE: When you run make down, it will clear all volumes except the data_mysqld volume. +> This is where your database and other persisted data is stored. +> If you want to start fresh, you can delete the data_mysqld volume. + +```sh +make down +make docker_mysqld_volume_remove # Remove the mysql database volume +make up +``` + +If you want to completely nuke your environment and start over as if you had just cloned the repo, +you can run `make clean_docker`. This will `make down` and remove all docker resources taking space on the host machine. + +### Accessing the Development App + +- Add the following entry to your `/etc/hosts` file to access **addons-server** via a local domain: + + ```sh + 127.0.0.1 olympia.test + ``` + +- The web application should now be accessible at `http://olympia.test`. +- You can access the web container for debugging and development: + + ```sh + make shell + ``` + +- To access the Django shell within the container: + + ```sh + make djshell + ``` + +## Configuring your environment + +Addons-server runs via [docker-compose](./building_and_running_services.md) and can be run in a local environment or on CI. +It is highly configurable to meet the requirements for different environments and use cases. +Here are some practical ways you can configure how _addons-server_ runs. + +### Build vs Pull + +By default, _addons-server_ builds a [docker image](./building_and_running_services.md) tagged _local_ before running the containers as a part of `make up`. +To run _addons-server_ with the _local_ image, just run `make up` like you normally would. It is the default. + +Instead of building, you can configure your environment to run a pulled image instead. To run a pulled image, +specify a {ref}`version or digest ` when calling `make up`. E.g `make up DOCKER_VERSION=latest` to run +the latest published version of `addons-server`. + +For typical development it is recommended to use the default built image. It is aggressively cached and most closely +reflects the current state of your local repository. Pulling a published image can be useful if you have limited CPU +or if you want to run a very specific version of addons-server for testing a Pull request +or debugging a currently deployed version. + +(version-vs-digest)= +### Version vs Digest + +The default behavior is to build the docker image locally, but if you want to run addons-server with a remote image +you can specify a docker image version to pull with: + +```bash +make up DOCKER_VERSION= +``` + +Version is the published tag of addons-server and corresponds to `mozilla/addons-server:` in [dockerhub][addons-server-tags]. + +> **Important**: When using a remote image (via `DOCKER_VERSION` or `DOCKER_DIGEST`), the `DOCKER_TARGET` must be set to 'production'. +> Running remote images in development mode is not supported and will fail validation during setup. + +Specifying a version will configure docker compose to set the [pull policy] to _always_ and specify the _image_ property +in the docker compose config to pull the latest build of the specified `version`. Once you've specified a version, +subsequent calls to `make up` will pull the same version consistently {ref}`see idempotence ` for more details. + +What if you want to run an exact build of `addons-server`, +without fetching later versions that might subsequently get published to the same tag? + +You can specify a `DOCKER_DIGEST` to pull a specific build of addons-server. This can be very useful if you want +to guarantee the exact state of the image you are running. This is used in our own CI environments to ensure each job +runs with the exact same image built in the run. + +```bash +make up DOCKER_DIGEST=sha256@abc123 +``` + +A docker [build digest][docker-image-digest] corresponds to the precise state of a docker image. +Think of it like a content hash, though it's a bit more complicated than that. +Specifying a build digest means you will always run the exact same version +of the image and it will not change the contents of the image. + +Our [CI][ci-workflow] workflow builds and pushes a docker image on each run. To run the exact image built during a CI run, +copy the image digest from the _build_ job logs. Look for a log line like this: + +```shell +#36 pushing manifest for docker.io/mozilla/addons-server:pr-22395-ci@sha256:8464804ed645e429ccb3585a50c6003fafd81bd43407d8d4ab575adb8391537d +``` + +The version for the above image is `pr-22395-ci` and the digest is `sha256:8464804ed645e429ccb3585a50c6003fafd81bd43407d8d4ab575adb8391537d`. +To run the specific build of the exact run for `pr-22395` you would run: + +```bash + make up DOCKER_VERSION=pr-22395-ci +``` + +And to run exactly the version built in this run, even if it is not the latest version, you would run: + +```bash + make up DOCKER_DIGEST=sha256:8464804ed645e429ccb3585a50c6003fafd81bd43407d8d4ab575adb8391537d +``` + +If you specify both a version and digest, digest as the more specific attribute takes precedence. + +(idempotence)= +### Idempotence + +The `make up` command and all of its sub-commands are idempotent. +That means if the command is repeated with the same inputs you will always get the same result. +If you run + +```bash + make up DOCKER_VERSION=banana +``` + +and then run make up again, the .env file will have a docker tag specifying the version `banana`. +This prevents you from needing to constantly specify parameters over and over. +But it also means you have to remember what values you have set for different properties as they can have huge +impacts on what is actually running in your environment. + +`make up` logs the current environment specifications to the terminal as it is running so you should always know +what exactly is happening in your environment at any given time. + +Additionally, by defining all of the critical docker compose variables in a .env file, it means that the behaviour +of running commands via `make` or running the same command directly via the docker CLI should produce the same result. + +Though it is **highly recommended to use the make commands** instead of directly calling docker in your terminal. + +### Docker Compose Files + +- **[docker-compose.yml][docker-compose]**: The primary Docker Compose file defining services, networks, and volumes for local and CI environments. +- **[docker-compose.private.yml][docker-compose-private]**: Runs addons-server with the _customs_ service that is only available to Mozilla employees + +Our docker compose files rely on substituted values, all of which are included in our .env file for direct CLI compatibility. +Any referenced _${VARIABLE}_ in the docker-compose files will be replaced with the value from the .env file. We have tests +that ensure any references are included in the .env file with valid values. + +This means when you run `make docker_compose_up`, the output on your machine will be exactly the same as if you ran +`docker compose up -d --wait --remove-orphans --force-recreate --quiet-pull` directly. You **should** use make commands, +but sometimes you need to debug further what a command is running on the terminal and this architecture allows you to do that. + +By following these steps, you can set up your local development environment and understand the existing CI workflows for the **addons-server** project. For more details on specific commands and configurations, refer to the upcoming sections in this documentation. + +### Environment Validation + +The setup process includes strict validation of environment variables to ensure a valid configuration: + +| Validation Rule | Description | Required | Validation | Default | From .env file| +|----------------|-------------|----------|------------|---------|--------------| +| `DOCKER_TAG` | The full docker tag for the image | false | (derived from other values) | mozilla/addons-server:local |true | +| `DOCKER_TARGET` | The target stage to build the docker image to | true | must be `production` when building an image or using a remote image | `development` for local images, `production` for remote images | true (only for local images) | +| `DOCKER_VERSION` | The docker tag version to use when building/pulling an image | false | none | local | false | +| `DOCKER_DIGEST` | The docker image build digest to pull a specific build | false | overrides `DOCKER_VERSION` | none | false | + +These validations help prevent configuration issues early in the setup process. + +## Gotchas + +Here's a list of a few of the issues you might face when setting up your development environment + +### Can't access the web server? + +Check you've created a hosts file entry pointing `olympia.test` to the relevant IP address. + +If containers are failing to start use `docker compose ps` to check their running status. + +Another way to find out what's wrong is to run `docker compose logs`. + +### Getting "Programming error [table] doesn't exist"? + +Make sure you've run `make up`. + +### ConnectionError during initialize (elasticsearch container fails to start) + +When running `make up` without a working elasticsearch container, you'll get a ConnectionError. Check the logs with `docker compose logs`. If elasticsearch is complaining about `vm.max_map_count`, run this command on your computer or your docker-machine VM: + +```sh + sudo sysctl -w vm.max_map_count=262144 +``` + +This allows processes to allocate more [memory map areas](https://stackoverflow.com/a/11685165/4496684). + +### Connection to elasticsearch timed out (elasticsearch container exits with code 137) + +`docker compose up -d` brings up all containers, but running `make up` causes the elasticsearch container to go down. Running `docker compose ps` shows _Exited (137)_ against it. + +Update default settings in Docker Desktop - we suggest increasing RAM limit to at least 4 GB in the Resources/Advanced section and click on "Apply and Restart". + +### Port collisions (nginx container fails to start) + +If you're already running a service on port 80 or 8000 on your host machine, the _nginx_ container will fail to start. This is because the `docker-compose.override.yml` file tells _nginx_ to listen on port 80 and the web service to listen on port 8000 by default. + +This problem will manifest itself by the services failing to start. Here's an example for the most common case of _nginx_ not starting due to a collision on port 80: + +```shell + ERROR: for nginx Cannot start service nginx:..... + ...Error starting userland proxy: Bind for 0.0.0.0:80: unexpected error (Failure EADDRINUSE) + ERROR: Encountered errors while bringing up the project. +``` + +You can check what's running on that port by using (sudo is required if you're looking at port < 1024): + +```sh + sudo lsof -i :80 +``` + +We specify the ports _nginx_ listens on in the `docker-compose.override.yml` file. If you wish to override the ports you can do so by creating a new _docker-compose_ config and starting the containers using that config alongside the default config. + +For example, if you create a file called `docker-compose-ports.yml`: + +```yaml + nginx: + ports: + - 8880:80 +``` + +Next, you would stop and start the containers with the following: + +```sh + docker compose stop # only needed if running + docker compose -f docker-compose.yml -f docker-compose-ports.yml up -d +``` + +Now the container _nginx_ is listening on 8880 on the host. You can now proxy to the container _nginx_ from the host _nginx_ with the following _nginx_ config: + +```nginx + server { + listen 80; + server_name olympia.test; + location / { + proxy_pass http://olympia.test:8880; + } + } +``` + +### returned Internal Server Error for API route and version + +This can occur if the docker daemon has crashed. Running docker commands will return errors as the CLI cannot communicate +with the daemon. The best thing to do is to restart docker and to check your docker memory usage. The most likely cause +is limited memory. You can check the make commands to see how you can free up space on your machine. + +```bash + docker volume create addons-server_data_mysqld + request returned Internal Server Error for API route and version http://%2FUsers%2Fwilliam%2F.docker%2Frun%2Fdocker.sock/v1.45/volumes/create, check if the server supports the requested API version + make: *** [docker_mysqld_volume_create] Error 1 +``` + +### Mysqld failing to start + +Our MYSQLD service relies on a persistent data volume in order to save the database even after containers are removed. +It is possible that the volume is in an incorrect state during startup which can lead to erros like the following: + +```bash + mysqld-1 | 2024-06-14T13:50:33.169411Z 0 [ERROR] [MY-010457] [Server] --initialize specified but the data directory has files in it. Aborting. + mysqld-1 | 2024-06-14T13:50:33.169416Z 0 [ERROR] [MY-013236] [Server] The designated data directory /var/lib/mysql/ is unusable. You can remove all files that the server added to it. +``` + +The best way around this is to `make down && make up` This will prune volumes and restart addons-server. + +### stat /Users/kmeinhardt/src/mozilla/addons-server/env: no such file or directory + +If you ran into this issue, it is likely due to an invalid .env likely created via running tests for our makefile +and docker-comose.yml file locally. + +```bash + docker compose up -d --wait --remove-orphans --force-recreate --quiet-pull + stat /Users/kmeinhardt/src/mozilla/addons-server/env: no such file or directory + make: *** [docker_compose_up] Error 14 +``` + +To fix this error `rm -f .env` to remove your .env and `make up` to restart the containers. + +[docker-compose]: ../../../docker-compose.yml +[docker-compose-private]: ../../../docker-compose.private.yml +[docker-image-digest]: https://github.com/opencontainers/.github/blob/main/docs/docs/introduction/digests.md +[addons-server-tags]: https://hub.docker.com/r/mozilla/addons-server/tags +[ci-workflow]: https://github.com/mozilla/addons-server/actions/workflows/ci.yml + +### 401 during docker build step in CI + +If the `build-docker` action is run it requires repository secret and permissions to be set correctly. If you see the below error: + +```bash +Error: buildx bake failed with: ERROR: failed to solve: failed to push mozilla/addons-server:pr-22446-ci: failed to authorize: failed to fetch oauth token: unexpected status from GET request to https://auth.docker.io/token?scope=repository%3Amozilla%2Faddons-server%3Apull%2Cpush&service=registry.docker.io: 401 Unauthorized +``` + +See the (workflow example)[./github_actions.md] for correct usage. + +### Invalid pull_policy + +If you run docker compose commands directly in the terminal, it is critical that your `.env` file exists and is up to date. This is handled automatically using make commands +but if you run `docker compose pull` without a .env file, you may encounter validation errors. That is because our docker-compose file itself uses variable substituation +for certain properties. This allows us to modify the behaviour of docker at runtime. + +```bash +validating /Users/user/mozilla/addons-server/docker-compose.yml: services.worker.pull_policy services.worker.pull_policy must be one of the following: "always", "never", "if_not_present", "build", "missing" +``` + +To fix this error, run `make setup` to ensure you have an up-to-date .env file locally. + +### Invalid docker context + +We have in the past used custom docker build contexts to build and run addons-server. +We currently use the `default` builder context so if you get this error running make up: + +```bash +ERROR: run `docker context use default` to switch to default context +18306 v0.16.1-desktop.1 /Users/awagner/.docker/cli-plugins/docker-buildx buildx use default +github.com/docker/buildx/commands.runUse + github.com/docker/buildx/commands/use.go:31 +github.com/docker/buildx/commands.useCmd.func1 + github.com/docker/buildx/commands/use.go:73 +github.com/docker/cli/cli-plugins/plugin.RunPlugin.func1.1.2 + github.com/docker/cli@v27.0.3+incompatible/cli-plugins/plugin/plugin.go:64 +github.com/spf13/cobra.(*Command).execute + github.com/spf13/cobra@v1.8.1/command.go:985 +github.com/spf13/cobra.(*Command).ExecuteC + github.com/spf13/cobra@v1.8.1/command.go:1117 +github.com/spf13/cobra.(*Command).Execute + github.com/spf13/cobra@v1.8.1/command.go:1041 +github.com/docker/cli/cli-plugins/plugin.RunPlugin + github.com/docker/cli@v27.0.3+incompatible/cli-plugins/plugin/plugin.go:79 +main.runPlugin + github.com/docker/buildx/cmd/buildx/main.go:67 +main.main + github.com/docker/buildx/cmd/buildx/main.go:84 +runtime.main + runtime/proc.go:271 +runtime.goexit + runtime/asm_arm64.s:1222 + +make[1]: *** [docker_use_builder] Error 1 +make: *** [docker_pull_or_build] Error 2 +``` + +To fix this error, run `docker context use default` to switch to the default builder context. + +### Failing make up due to invalid or failing migrations + +Every time you run `make up` it will run migrations. If you have failing migrations, +this will cause the make command to fail. However, if migrations are running, it means the containers are already up. + +You can inspect and fix the migration and then run `make up` again to start the re-start the containers. + +Inspecting the database can be done via: + +```bash +make dbshell +``` + +### Environment validation errors during setup + +If you see validation errors during `make up` like this: + +```bash +Invalid items: check setup.py for validations +• DOCKER_TARGET +• DOCKER_COMMIT +• DOCKER_BUILD +``` + +This usually means you're trying to use a remote image (via `DOCKER_VERSION` or `DOCKER_DIGEST`) without the required production configuration. Remember that remote images: + +- Must use `DOCKER_TARGET=production` +- Require `DOCKER_COMMIT` and `DOCKER_BUILD` values + +For local development, use the default local image build which has more flexible validation rules. diff --git a/docs/topics/development/static-files.md b/docs/topics/development/static-files.md new file mode 100644 index 000000000000..00b6d16f2a18 --- /dev/null +++ b/docs/topics/development/static-files.md @@ -0,0 +1,86 @@ +# Static Files in addons-server + +This document explains how static files are served in the addons-server project during local development. In production, +static files are served directly from a CDN. + +## Overview + +addons-server uses a combination of nginx and Django's built-in static file serving capabilities to efficiently serve static files. +These files come from multiple sources: + +1. The `./static` folder in the project +2. Python dependencies +3. npm dependencies +4. Compressed/minified files built by `update_assets` + +## Static File Servers + +We use a combination of servers to serve static files: + +1. Nginx +2. Django's built-in development server + +In development, the nginx server will attempt to serve static files from the `./static` directory mounted into the nginx cointainer. +If the file cannot be found there the request is forwarded to django. +Nginx serves our own static files quickly and any vendor files can be fetched from django directly during development. + +In production mode, we mount a data volume both to `web` anb `nginx` containers. +The `web` container exposes the `site-static` directory to nginx that includes the collected static files. + +> In actual production environments, we upload the static files to a cloud bucket and serve them directly from the static path. + +## Static File Sources + +The rendering path for static files is as follows: + +1. Nginx tries to serve the file if it is available in the `./static` directory. +2. If the file is not found, the request is forwarded to django and served by the static file server. + +The static file serve uses our defined `STATICFILES_STORAGE` setting to determine the URL for static files as well as their underlying source file. +During development, we use the `StaticFilesStorage` class which does not map the hashed file names back to their original file names. +Otherwise we use the same `ManifestStaticFilesStorage` class that is used in production, expecting to serve the files from the `STATIC_ROOT` directory. + +This allows us to skip `update_assets` in dev mode, speeding up the development process, while still enabling production-like behavior +when configured to do so. The long term goal is to run CI in production mode always to ensure all tests verify against the production +static file build. + +To better visualize the impact of the various settings, here is a reference: + +Given a static file 'js/devhub/my-file.js': + +In `DEV_MODE` the url will look like `/static/js/devhub/my-file.js` no matter what. +However, in production, if `DEBUG` is `False`, the url will append the content hash like this, +`/static/js/devhub/my-file.1234567890.js`. Finally, if `DEBUG` is true, this file will be minified and concatenated with other files and probably look something like this `/static/js/devhub-all.min.1234567890.js`. + +The true `production` mode is then when `DEBUG` is `False` and `DEV_MODE` is `False`. But it makes sense +to make these individually toggleable so you can better "debug" js files from a production image. + +### Project Static Files + +Static files specific to the addons-server project are stored in the `./static` directory. These include CSS, JavaScript, images, and other assets used by the application. + +In reality there are 3 static directories in our docker compose container: + +- `/data/olympia/static`: Contains static files that are mounted directly from the host. +- `/data/olympia/static-build`: Contains static files that are built by `vite`. +- `/data/olympia/site-static`: Contains static files that are collected by the `collectstatic` command. + +The only of these directories that is exposed to your host is the `./static` directory. + +### Compressing Static Files + +We currently use a `ducktape` script to compress our static files. +Ideally we would migrate to a modern tool to replace manual scripting, but for now this works. + +Assets are compressed automatically during the docker build, but if you need to manually update files while developing, +the easiest way is to run `make update_assets` which will compress and concatenate static assets as well as collect all static files +to the `site-static` directory. + +### Python Dependencies + +Some Python packages include their own static files. These assets are collected by the `collectstatic` command and included in the final static files directory. +During development they are served by the django development server. + +### npm Dependencies + +Static assets are built by `vite` and served by nginx. diff --git a/docs/topics/development/style.rst b/docs/topics/development/style.md similarity index 52% rename from docs/topics/development/style.rst rename to docs/topics/development/style.md index db94d5b2b8b7..309e7cafab72 100644 --- a/docs/topics/development/style.rst +++ b/docs/topics/development/style.md @@ -1,52 +1,51 @@ -.. _style: +(style)= -=================== -Style Guide -=================== +# Style Guide Writing code for olympia? Awesome! Please help keep our code readable by, whenever possible, adhering to these style conventions. +## Python -Python ------- -- see https://wiki.mozilla.org/Webdev:Python +- see +## Markup -Markup ------- -- ```` +- `` - double-quote attributes - Soft tab (2 space) indentation -- Title-Case ``