diff --git a/.docker-mongo/Dockerfile b/.docker-mongo/Dockerfile index 0c2e69a1d385..f0e971bbe4b3 100644 --- a/.docker-mongo/Dockerfile +++ b/.docker-mongo/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12.21.0-buster-slim +FROM node:12.22.1-buster-slim LABEL maintainer="buildmaster@rocket.chat" diff --git a/.docker/Dockerfile b/.docker/Dockerfile index d549a3943c5c..0c55289a84ed 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12.21.0-buster-slim +FROM node:12.22.1-buster-slim LABEL maintainer="buildmaster@rocket.chat" diff --git a/.docker/Dockerfile.rhel b/.docker/Dockerfile.rhel index 70d2ab602bda..c18889406413 100644 --- a/.docker/Dockerfile.rhel +++ b/.docker/Dockerfile.rhel @@ -1,6 +1,6 @@ FROM registry.access.redhat.com/ubi8/nodejs-12 -ENV RC_VERSION 3.13.2 +ENV RC_VERSION 3.14.0 MAINTAINER buildmaster@rocket.chat diff --git a/.eslintignore b/.eslintignore index 6bb92334d4d9..507106b4cf97 100644 --- a/.eslintignore +++ b/.eslintignore @@ -18,5 +18,7 @@ imports/client/ !/.storybook/ ee/server/services/dist/** !/.mocharc.js +!/client/.eslintrc.js +!/ee/client/.eslintrc.js app/utils/client/lib/sha1.js app/analytics/server/trackEvents.js diff --git a/.github/history.json b/.github/history.json index 59c1a41ad184..2d3520884504 100644 --- a/.github/history.json +++ b/.github/history.json @@ -15363,14 +15363,6 @@ ] }, "HEAD": { - "node_version": "12.21.0", - "npm_version": "6.14.8", - "apps_engine_version": "1.24.1", - "mongo_versions": [ - "3.4", - "3.6", - "4.0" - ], "pull_requests": [] }, "0.66.0-rc.0": { @@ -58234,6 +58226,1277 @@ ] } ] + }, + "3.11.4": { + "node_version": "12.18.4", + "npm_version": "6.14.8", + "apps_engine_version": "1.22.2", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [] + }, + "3.12.4": { + "node_version": "12.18.4", + "npm_version": "6.14.8", + "apps_engine_version": "1.23.0", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [] + }, + "3.13.2": { + "node_version": "12.21.0", + "npm_version": "6.14.8", + "apps_engine_version": "1.24.1", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [ + { + "pr": "21570", + "title": "Release 3.13.2", + "userLogin": "sampaiodiego", + "contributors": [ + "sampaiodiego" + ] + } + ] + }, + "3.13.3": { + "node_version": "12.21.0", + "npm_version": "6.14.8", + "apps_engine_version": "1.24.1", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [ + { + "pr": "21491", + "title": "[FIX] Team's channels list for teams with too many channels", + "userLogin": "KevLehman", + "description": "- Fix teams.listRooms pagination for non-admin users", + "milestone": "3.13.3", + "contributors": [ + "KevLehman", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21644", + "title": "[FIX] Livechat not retrieving messages", + "userLogin": "cuonghuunguyen", + "milestone": "3.13.3", + "contributors": [ + "cuonghuunguyen" + ] + } + ] + }, + "3.11.5": { + "node_version": "12.18.4", + "npm_version": "6.14.8", + "apps_engine_version": "1.22.2", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [ + { + "pr": "21644", + "title": "[FIX] Livechat not retrieving messages", + "userLogin": "cuonghuunguyen", + "milestone": "3.13.3", + "contributors": [ + "cuonghuunguyen" + ] + } + ] + }, + "3.12.5": { + "node_version": "12.18.4", + "npm_version": "6.14.8", + "apps_engine_version": "1.23.0", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [ + { + "pr": "21644", + "title": "[FIX] Livechat not retrieving messages", + "userLogin": "cuonghuunguyen", + "milestone": "3.13.3", + "contributors": [ + "cuonghuunguyen" + ] + } + ] + }, + "3.14.0-rc.0": { + "node_version": "12.22.1", + "npm_version": "6.14.1", + "apps_engine_version": "1.25.0-alpha.4943", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [ + { + "pr": "21119", + "title": "[FIX] Allow deletion of own account for passwordless accounts (e.g. OAUTH)", + "userLogin": "wolbernd", + "milestone": "3.14.0", + "contributors": [ + "wolbernd", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21432", + "title": "[FIX] Send alternative color to unread sidebar icon", + "userLogin": "tiagoevanp", + "description": "![image](https://user-images.githubusercontent.com/17487063/113469819-08f76b00-9427-11eb-942e-783c186ba7cd.png)", + "milestone": "3.14.0", + "contributors": [ + "tiagoevanp", + "ggazzo", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21684", + "title": "[FIX] Show direct rooms as readonly when one of the users is deactivated", + "userLogin": "KevLehman", + "milestone": "3.14.0", + "contributors": [ + "KevLehman", + "web-flow", + "sampaiodiego" + ] + }, + { + "pr": "21694", + "title": "Bump Livechat Version", + "userLogin": "ggazzo", + "milestone": "3.14.0", + "contributors": [ + "ggazzo" + ] + }, + { + "pr": "21690", + "title": "[NEW][APPS] Method to fetch Livechat Departments", + "userLogin": "d-gubert", + "description": "New method in the livechat bridge that allows apps to fetch departments that are enabled and have agents assigned", + "milestone": "3.14.0", + "contributors": [ + "d-gubert" + ] + }, + { + "pr": "21353", + "title": "[IMPROVE][APPS] Scheduler option to skip immediate execution of recurring jobs", + "userLogin": "thassiov", + "description": "Create and schedule a task manually at `scheduleRecurring` method so the first iteration runs after the configured interval. This is accomplished by adding the setting `skipImmediate: true` when setting up the task.", + "milestone": "3.14.0", + "contributors": [ + "thassiov", + "web-flow", + "d-gubert", + "lolimay" + ] + }, + { + "pr": "21204", + "title": "Fix typo in app/apps/README file", + "userLogin": "sauravjoshi23", + "contributors": [ + "sauravjoshi23" + ] + }, + { + "pr": "21490", + "title": "[FIX] Avoid sidebar being broke", + "userLogin": "tiagoevanp", + "contributors": [ + "tiagoevanp", + "ggazzo", + "web-flow" + ] + }, + { + "pr": "21519", + "title": "Add ')' after Date and Time in DB migration", + "userLogin": "im-adithya", + "contributors": [ + "im-adithya", + "web-flow" + ] + }, + { + "pr": "21689", + "title": "[IMPROVE] add permission check when adding a channel to a team", + "userLogin": "g-thome", + "description": "add permission check for each room", + "contributors": [ + "g-thome", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21692", + "title": "[NEW][Enterprise] Second layer encryption for data transport (alpha)", + "userLogin": "rodrigok", + "description": "The second layer encryption for data transport works implementing the ECDH algorithm where session keys are exchanged before the rest of the communication. This feature is **enterprise only** since it requires the micro-services architecture and it's in the early stage of tests as an **alpha** feature and documentation may not be available before the beta stage.", + "milestone": "3.14.0", + "contributors": [ + "rodrigok", + "web-flow" + ] + }, + { + "pr": "21658", + "title": "[NEW][ENTERPRISE] LDAP Teams Sync", + "userLogin": "pierre-lehnen-rc", + "milestone": "3.14.0", + "contributors": [ + "pierre-lehnen-rc" + ] + }, + { + "pr": "21579", + "title": "[FIX] Message link null corrupts message rendering", + "userLogin": "g-thome", + "description": "Additional checks on message_link field before rendering message contents", + "contributors": [ + "g-thome" + ] + }, + { + "pr": "18357", + "title": "[NEW] Standard Importer Structure", + "userLogin": "pierre-lehnen-rc", + "milestone": "3.14.0", + "contributors": [ + "pierre-lehnen-rc" + ] + }, + { + "pr": "21607", + "title": "[NEW] Password history", + "userLogin": "matheusbsilva137", + "description": "- Store each user's previously used passwords in a `passwordHistory` field (in the `users` record);\r\n- Users' previously used passwords are stored in their `passwordHistory` even when the setting is disabled;\r\n- Add \"Password History\" setting -- when enabled, it blocks users from reusing their most recent passwords;\r\n- Convert `comparePassword` file to TypeScript.\r\n\r\n![Password_Change](https://user-images.githubusercontent.com/36537004/115035168-ac726200-9ea2-11eb-93c6-fc8182ba5f3f.png)\r\n![Password_History](https://user-images.githubusercontent.com/36537004/115035175-ad0af880-9ea2-11eb-9f40-94c6327a9854.png)", + "contributors": [ + "matheusbsilva137", + "web-flow" + ] + }, + { + "pr": "21686", + "title": "[IMPROVE] OEmbed details by requesting using the accept language header on the request", + "userLogin": "KevLehman", + "description": "- Send `Accept-Language` header on oembed requests", + "contributors": [ + "KevLehman" + ] + }, + { + "pr": "21656", + "title": "[FIX] User status out of sync", + "userLogin": "ggazzo", + "contributors": [ + "ggazzo" + ] + }, + { + "pr": "21657", + "title": "[FIX] Generic Attachment broken somehow", + "userLogin": "ggazzo", + "contributors": [ + "ggazzo" + ] + }, + { + "pr": "21565", + "title": "[NEW][APPS] onInstall and onUninstall events", + "userLogin": "lucassartor", + "description": "Adding the `user` information when installing and uninstalling an App to the Apps-Engine.", + "milestone": "3.14.0", + "contributors": [ + "lucassartor", + "web-flow", + "d-gubert" + ] + }, + { + "pr": "21360", + "title": "[NEW] On Hold system messages", + "userLogin": "murtaza98", + "description": "![image](https://user-images.githubusercontent.com/34130764/115442079-3a49a680-a22f-11eb-9ee8-6c705097cd57.png)", + "milestone": "3.14.0", + "contributors": [ + "murtaza98", + "web-flow", + "rafaelblink", + "renatobecker" + ] + }, + { + "pr": "21593", + "title": "[IMPROVE] Resize custom emojis on upload instead of saving at max res", + "userLogin": "KevLehman", + "description": "- Create new MediaService (ideally, should be in charge of all media-related operations)\r\n- Resize emojis to 128x128", + "contributors": [ + "KevLehman" + ] + }, + { + "pr": "21513", + "title": "[FIX] Rename Omnichannel Rooms, Inquiries and Subscriptions when the Contact Name changes", + "userLogin": "rafaelblink", + "milestone": "3.14.0", + "contributors": [ + "rafaelblink", + "web-flow", + "renatobecker" + ] + }, + { + "pr": "21495", + "title": "[FIX] public teams not appearing on spotlight search results", + "userLogin": "pierre-lehnen-rc", + "milestone": "3.14.0", + "contributors": [ + "pierre-lehnen-rc", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21016", + "title": "[IMPROVE] Add error messages to the creation of channels or usernames containing reserved words", + "userLogin": "matheusbsilva137", + "description": "Display error messages when the user attempts to create or edit users' or channels' names with any of the following words (**case-insensitive**):\r\n- admin;\r\n- administrator;\r\n- system;\r\n- user.\r\n![create-channel](https://user-images.githubusercontent.com/36537004/110132223-b421ef80-7da9-11eb-82bc-f0d4e1df967f.png)\r\n![register-username](https://user-images.githubusercontent.com/36537004/110132234-b71ce000-7da9-11eb-904e-580233625951.png)\r\n![change-channel](https://user-images.githubusercontent.com/36537004/110143057-96f31e00-7db5-11eb-994a-39ae9e63392e.png)\r\n![change-username](https://user-images.githubusercontent.com/36537004/110143065-98244b00-7db5-11eb-9d13-afc5dc9866de.png)", + "milestone": "3.14.0", + "contributors": [ + "matheusbsilva137", + "web-flow", + "sampaiodiego" + ] + }, + { + "pr": "21509", + "title": "[FIX] Remove all agent subscriptions when an Omnichannel chat is closed", + "userLogin": "renatobecker", + "milestone": "3.14.0", + "contributors": [ + "renatobecker", + "web-flow" + ] + }, + { + "pr": "21417", + "title": "regression: Team Channels actions", + "userLogin": "gabriellsh", + "contributors": [ + "gabriellsh", + "web-flow" + ] + }, + { + "pr": "21682", + "title": "[FIX] Wrong title on Omnichannel contact information panel", + "userLogin": "rafaelblink", + "milestone": "3.14.0", + "contributors": [ + "rafaelblink", + "web-flow" + ] + }, + { + "pr": "21488", + "title": "[IMPROVE] Do not require pre-configured tags in Omnichannel chats", + "userLogin": "rafaelblink", + "milestone": "3.14.0", + "contributors": [ + "rafaelblink", + "web-flow", + "renatobecker" + ] + }, + { + "pr": "21457", + "title": "[FIX] Margins on contextual bar information", + "userLogin": "dougfabris", + "description": "### Room\r\n**Before**\r\n![image](https://user-images.githubusercontent.com/27704687/115080812-ba8fa500-9ed9-11eb-9078-3625603bf92b.png)\r\n\r\n**After**\r\n![image](https://user-images.githubusercontent.com/27704687/115080966-e9a61680-9ed9-11eb-929f-6516c1563e99.png)\r\n\r\n### Livechat\r\n![image](https://user-images.githubusercontent.com/27704687/113640101-1859fc80-9651-11eb-88f8-09a899953988.png)", + "contributors": [ + "dougfabris" + ] + }, + { + "pr": "21511", + "title": "[FIX] Allows more than 25 discussions/files to be loaded in the contextualbar", + "userLogin": "Jeanstaquet", + "description": "In some places, you could not load more than 25 threads/discussions/files on the screen when searching the lists in the contextualbar.\r\nThreads & list are numbered for a better view of the solution\r\n\r\n\r\nhttps://user-images.githubusercontent.com/45966964/114222225-93335800-996e-11eb-833f-568e83129aae.mp4", + "contributors": [ + "Jeanstaquet", + "web-flow" + ] + }, + { + "pr": "21669", + "title": "[FIX] Selected channels are not showing in Teams", + "userLogin": "sumukhah", + "contributors": [ + "sumukhah" + ] + }, + { + "pr": "21598", + "title": "Regression: Legacy Banner Position", + "userLogin": "dougfabris", + "description": "### Before:\r\n![image](https://user-images.githubusercontent.com/27704687/114961773-dc3c4e00-9e3f-11eb-9a32-e882db3fbfbc.png)\r\n\r\n### After\r\n![image](https://user-images.githubusercontent.com/27704687/114961673-a6976500-9e3f-11eb-9238-a12870d7db8f.png)", + "contributors": [ + "dougfabris", + "ggazzo" + ] + }, + { + "pr": "21428", + "title": "[FIX] Remove size prop from StatusBullet component", + "userLogin": "tiagoevanp", + "contributors": [ + "tiagoevanp", + "ggazzo", + "web-flow" + ] + }, + { + "pr": "21466", + "title": "[FIX] Audio message same pattern as image message", + "userLogin": "tiagoevanp", + "description": "![image](https://user-images.githubusercontent.com/17487063/113760168-4c363000-96ec-11eb-9138-0fbcedb3fa42.png)", + "contributors": [ + "tiagoevanp", + "ggazzo" + ] + }, + { + "pr": "21518", + "title": "[FIX] Allows to display more than 25 users maximum in the users list", + "userLogin": "Jeanstaquet", + "description": "Now when you scroll to the bottom of the users list, it shows more users. Before the fix, the limit for the query for loadMore was calculated so that no additional users could be loaded.\r\n\r\nBefore\r\n\r\nhttps://user-images.githubusercontent.com/45966964/114249739-baece500-999b-11eb-9bb0-3a5bcee18ad8.mp4\r\n\r\nAfter\r\n\r\n\r\nhttps://user-images.githubusercontent.com/45966964/114249895-364e9680-999c-11eb-985c-47aedc763488.mp4", + "contributors": [ + "Jeanstaquet", + "web-flow" + ] + }, + { + "pr": "21508", + "title": "[FIX] Allows more than 25 threads to be loaded, fixes #21507", + "userLogin": "Jeanstaquet", + "contributors": [ + "Jeanstaquet", + "web-flow", + "MartinSchoeler" + ] + }, + { + "pr": "21534", + "title": "[FIX] Use async await in TeamChannels delete channel action", + "userLogin": "tiagoevanp", + "contributors": [ + "tiagoevanp" + ] + }, + { + "pr": "21617", + "title": "[IMPROVE] Alert on team deletion", + "userLogin": "MartinSchoeler", + "description": "\"Screen", + "contributors": [ + "MartinSchoeler" + ] + }, + { + "pr": "21612", + "title": "[FIX] Team types in admin -> rooms.", + "userLogin": "gabriellsh", + "description": "![print](https://user-images.githubusercontent.com/40830821/115068327-82339b00-9ec8-11eb-8e37-726baf9d2db0.jpg)", + "contributors": [ + "gabriellsh" + ] + }, + { + "pr": "21650", + "title": "regression: Cannot enable e2e in direct room.", + "userLogin": "gabriellsh", + "contributors": [ + "gabriellsh" + ] + }, + { + "pr": "21535", + "title": "[FIX] Change team private info text", + "userLogin": "tiagoevanp", + "contributors": [ + "tiagoevanp" + ] + }, + { + "pr": "21461", + "title": "[FIX] Change margin size for quote messages", + "userLogin": "tiagoevanp", + "description": "![image](https://user-images.githubusercontent.com/17487063/113723723-02d3e980-96c8-11eb-9bc7-70aab5ea8091.png)", + "contributors": [ + "tiagoevanp" + ] + }, + { + "pr": "21416", + "title": "[FIX] Change the active appearance for toolbox buttons", + "userLogin": "tiagoevanp", + "description": "![image](https://user-images.githubusercontent.com/17487063/113359447-2d1b5500-931e-11eb-81fa-86f60fcee3a9.png)", + "contributors": [ + "tiagoevanp", + "web-flow" + ] + }, + { + "pr": "21491", + "title": "[FIX] Team's channels list for teams with too many channels", + "userLogin": "KevLehman", + "description": "- Fix teams.listRooms pagination for non-admin users", + "milestone": "3.13.3", + "contributors": [ + "KevLehman", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21552", + "title": "[FIX] Rename team not working properly", + "userLogin": "pierre-lehnen-rc", + "contributors": [ + "pierre-lehnen-rc", + "sampaiodiego" + ] + }, + { + "pr": "21642", + "title": "Language update from LingoHub 🤖 on 2021-04-19Z", + "userLogin": "lingohub[bot]", + "contributors": [ + null + ] + }, + { + "pr": "21644", + "title": "[FIX] Livechat not retrieving messages", + "userLogin": "cuonghuunguyen", + "milestone": "3.13.3", + "contributors": [ + "cuonghuunguyen" + ] + }, + { + "pr": "21561", + "title": "[Improve] Remove useless tabbar options from Omnichannel rooms", + "userLogin": "rafaelblink", + "milestone": "3.14.0", + "contributors": [ + "rafaelblink", + "renatobecker", + "web-flow" + ] + }, + { + "pr": "21616", + "title": "[FIX] Omnichannel current chats and agents grid aren't sorting by status properly", + "userLogin": "rafaelblink", + "milestone": "3.14.0", + "contributors": [ + "rafaelblink", + "web-flow" + ] + }, + { + "pr": "21134", + "title": "[NEW] REST endpoint `teams.update`", + "userLogin": "g-thome", + "description": "add teams.update endpoint", + "contributors": [ + "sampaiodiego", + "g-thome", + "KevLehman", + "web-flow" + ] + }, + { + "pr": "21613", + "title": "Regression: Edit user in admin breaking", + "userLogin": "gabriellsh", + "contributors": [ + "gabriellsh" + ] + }, + { + "pr": "21611", + "title": "Fix: Missing module `eventemitter3` for micro services", + "userLogin": "rodrigok", + "description": "- Fix error when running micro services after version 3.12\r\n- Fix build of docker image version latest for micro services", + "milestone": "3.14.0", + "contributors": [ + "rodrigok" + ] + }, + { + "pr": "21608", + "title": "[FIX] Omnichannel room information panel breaking due to lack of data verification", + "userLogin": "rafaelblink", + "milestone": "3.14.0", + "contributors": [ + "rafaelblink" + ] + }, + { + "pr": "21567", + "title": "Regression: React + Blaze reconciliation ", + "userLogin": "ggazzo", + "contributors": [ + "ggazzo", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21451", + "title": "[FIX] Wrong user in user info", + "userLogin": "gabriellsh", + "description": "Fixed some race conditions in admin.\r\n\r\nSelf DMs used to be created with the userId duplicated. Sometimes rooms can have 2 equal uids, but it's a self DM. Fixed a getter so this isn't a problem anymore.", + "contributors": [ + "gabriellsh" + ] + }, + { + "pr": "21525", + "title": "[FIX] Typos/missing elements in the French translation", + "userLogin": "Jeanstaquet", + "description": "- I have corrected some typos in the translation\r\n- I added a translation for missing words\r\n- I took the opportunity to correct a mistranslated word\r\n- Test_Desktop_Notifications was missing in the EN and FR file\r\n![image](https://user-images.githubusercontent.com/45966964/114290186-e7792d80-9a7d-11eb-8164-3b5e72e93703.png)", + "contributors": [ + "Jeanstaquet", + "web-flow" + ] + }, + { + "pr": "21563", + "title": "[FIX] Archive permissions for room moderator", + "userLogin": "tiagoevanp", + "contributors": [ + "tiagoevanp" + ] + }, + { + "pr": "21564", + "title": "[FIX] Checking 'start-discussion' Permission for MessageBox Actions", + "userLogin": "yash-rajpal", + "description": "Permissions 'start-discussion-other-user' and 'start-discussion' are checked everywhere before letting anyone start any discussions, this permission check was missing for message box actions, so added it.", + "contributors": [ + "yash-rajpal" + ] + }, + { + "pr": "21556", + "title": "[FIX] Correcting the case there are no result in admin users list ", + "userLogin": "Jeanstaquet", + "description": "I added a default case to the total when there are no result to the user's query", + "contributors": [ + "Jeanstaquet", + "web-flow" + ] + }, + { + "pr": "21483", + "title": "[FIX] Don't allow whitespace on bold, italic and strike", + "userLogin": "MartinSchoeler", + "description": "Stops the original markdown rendered from rendering empty bold, italic and strike text. Stops `_ _`, `* *` and `~ ~`", + "contributors": [ + "MartinSchoeler" + ] + }, + { + "pr": "21464", + "title": "[FIX] Message Block ordering ", + "userLogin": "gabriellsh", + "description": "Reactions should come before reply button.\r\n![image](https://user-images.githubusercontent.com/40830821/113748926-6f0e1780-96df-11eb-93a5-ddcfa891413e.png)", + "contributors": [ + "gabriellsh" + ] + }, + { + "pr": "20998", + "title": "[IMPROVE] Add proxy for data export", + "userLogin": "r0zbot", + "description": "Add a proxy for data export downloads (instead of just linking ufs urls) so we can have more control over its response. Also added a human readable message when the user tries to download the user-data unauthenticated.", + "contributors": [ + "r0zbot", + "sampaiodiego" + ] + }, + { + "pr": "21489", + "title": "[FIX] Updating a message causing URLs to be parsed even within markdown code", + "userLogin": "KevLehman", + "description": "- Fix `updateMessage` to avoid parsing URLs inside markdown\r\n- Honor `parseUrls` property when updating messages", + "contributors": [ + "KevLehman" + ] + }, + { + "pr": "21557", + "title": "[FIX] Fix the bugs opening discussions", + "userLogin": "Jeanstaquet", + "description": "I added the right row export to display the discussions list", + "contributors": [ + "Jeanstaquet", + "web-flow" + ] + }, + { + "pr": "21527", + "title": "A React-based replacement for BlazeLayout", + "userLogin": "tassoevan", + "description": "- The Meteor package **`kadira:blaze-layout` was removed**;\r\n- A **global subscription** for the current application layout (**`appLayout`**) replaces `BlazeLayout` entirely;\r\n- The **`#react-root` element** is rendered on server-side instead of dynamically injected into the DOM tree;\r\n- The **\"page loading\" throbber** is now rendered on the React tree;\r\n- The **`renderRouteComponent` helper was removed**;\r\n- Some code run without any criteria on **`main` template** module was moved into **client startup modules**;\r\n- React portals used to embed Blaze templates have their own subscription (**`blazePortals`**);\r\n- Some **route components were refactored** to remove a URL path trap originally disabled by `renderRouteComponent`;\r\n- A new component to embed the DOM nodes generated by **`RoomManager`** was created.", + "milestone": "3.14.0", + "contributors": [ + "tassoevan" + ] + }, + { + "pr": "21530", + "title": "Language update from LingoHub 🤖 on 2021-04-12Z", + "userLogin": "lingohub[bot]", + "contributors": [ + null, + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21482", + "title": "Chore: Increase testing coverage on password policy class", + "userLogin": "KevLehman", + "contributors": [ + "KevLehman" + ] + }, + { + "pr": "21494", + "title": "Chore: Meteor update to 2.1.1", + "userLogin": "sampaiodiego", + "description": "Basically Node update to version 12.22.1\r\n\r\nMeteor change log https://github.com/meteor/meteor/blob/devel/History.md#v211-2021-04-06", + "contributors": [ + "sampaiodiego" + ] + }, + { + "pr": "21484", + "title": "Chore: Do not stop animations on Test Mode", + "userLogin": "rodrigok", + "contributors": [ + "rodrigok" + ] + }, + { + "pr": "21493", + "title": "Chore: Remove control character from room model operation", + "userLogin": "pierre-lehnen-rc", + "contributors": [ + "pierre-lehnen-rc" + ] + }, + { + "pr": "21318", + "title": "[NEW] New set of rules for client code", + "userLogin": "tassoevan", + "description": "This _small_ PR does the following:\r\n\r\n- Now **React** is the web client's first-class citizen, being **loaded before Blaze**. Thus, `BlazeLayout` calls render templates inside of a React component (`BlazeLayoutWrapper`);\r\n- Main client startup code, including polyfills, is written in **TypeScript**;\r\n- At the moment, routes are treated as regular startup code; it's expected that `FlowRouter` will be deprecated in favor of a new routing library;\r\n- **React** was updated to major version **17**, deprecating the usage of `React` as namespace (e.g. use `memo()` instead of `React.memo()`);\r\n- The `client/` and `ee/client/` directory are linted with a **custom ESLint configuration** that includes:\r\n - **Prettier**;\r\n - `react-hooks/*` rules for TypeScript files;\r\n - `react/no-multi-comp`, enforcing the rule of **one single React component per module**;\r\n - `react/display-name`, which enforces that **React components must have a name for debugging**;\r\n - `import/named`, avoiding broken named imports.\r\n- A bunch of components were refactored to match the new ESLint rules.", + "milestone": "3.14.0", + "contributors": [ + "tassoevan" + ] + }, + { + "pr": "21465", + "title": "[FIX] Header component breaking if user is not part of teams room.", + "userLogin": "gabriellsh", + "milestone": "3.13.1", + "contributors": [ + "gabriellsh", + "web-flow" + ] + }, + { + "pr": "21469", + "title": "[FIX] Admin Users list pagination", + "userLogin": "KevLehman", + "description": "- Fix Administration/Users pagination", + "milestone": "3.13.1", + "contributors": [ + "KevLehman" + ] + }, + { + "pr": "21470", + "title": "[FIX] App installation from marketplace not correctly displaying the permissions", + "userLogin": "graywolf336", + "description": "Fixes the marketplace app installation not correctly displaying the permissions modal.", + "milestone": "3.13.1", + "contributors": [ + "graywolf336", + "d-gubert", + "web-flow", + "thassiov" + ] + }, + { + "pr": "21485", + "title": "[FIX] Omnichannel queue manager returning outdated room object", + "userLogin": "renatobecker", + "description": "The Omnichannel Queue Manager is returning outdated room object when delegating the chat to an agent, hence, our Livechat widget is affected and the agent assigned to the chat is not displayed on the widget, only after refreshing/reloading.", + "milestone": "3.13.1", + "contributors": [ + "renatobecker" + ] + }, + { + "pr": "21481", + "title": "[FIX] Close chat button is not available for Omnichannel agents", + "userLogin": "rafaelblink", + "milestone": "3.13.1", + "contributors": [ + "rafaelblink", + "renatobecker", + "web-flow" + ] + }, + { + "pr": "21476", + "title": "[FIX] Make Omnichannel's closing chat button the last action in the toolbox", + "userLogin": "rafaelblink", + "milestone": "3.13.1", + "contributors": [ + "rafaelblink", + "renatobecker", + "web-flow" + ] + }, + { + "pr": "21463", + "title": "[IMPROVE] Add support to range downloads on file system storage", + "userLogin": "sampaiodiego", + "milestone": "3.14.0", + "contributors": [ + "sampaiodiego" + ] + }, + { + "pr": "21454", + "title": "[FIX] Don't ask again modals blinking", + "userLogin": "gabriellsh", + "description": "Made the check before opening the modal.", + "contributors": [ + "gabriellsh", + "web-flow" + ] + }, + { + "pr": "21450", + "title": "[FIX] Error when editing Omnichannel rooms without custom fields", + "userLogin": "rafaelblink", + "milestone": "3.13.1", + "contributors": [ + "rafaelblink", + "renatobecker", + "web-flow" + ] + }, + { + "pr": "21453", + "title": "[FIX] Wrong useMemo on Priorities EE field.", + "userLogin": "rafaelblink", + "milestone": "3.13.1", + "contributors": [ + "rafaelblink", + "renatobecker", + "web-flow" + ] + }, + { + "pr": "21462", + "title": "[FIX] Add tag input to Closing Chat modal", + "userLogin": "rafaelblink", + "milestone": "3.13.1", + "contributors": [ + "rafaelblink" + ] + }, + { + "pr": "20478", + "title": " Doc: Corrected links to documentation of rocket.chat README.md ", + "userLogin": "joshi008", + "description": "The link for documentation in the readme was previously https://rocket.chat/docs/ while that was not working and according to the website it was https://docs.rocket.chat/\r\nThe link for deployment methods in readme was corrected from https://rocket.chat/docs/installation/paas-deployments/ to https://docs.rocket.chat/installation/paas-deployments\r\nSome more links to the documentations were giving 404 error which hence updated.", + "contributors": [ + "joshi008", + "web-flow" + ] + }, + { + "pr": "21446", + "title": "Language update from LingoHub 🤖 on 2021-04-05Z", + "userLogin": "lingohub[bot]", + "contributors": [ + null, + "sampaiodiego" + ] + }, + { + "pr": "21429", + "title": "[FIX] Tag component is no longer rendering on Chat Room Information panel", + "userLogin": "renatobecker", + "milestone": "3.14.0", + "contributors": [ + "renatobecker", + "rafaelblink", + "web-flow" + ] + }, + { + "pr": "21441", + "title": "Merge master into develop & Set version to 3.14.0-develop", + "userLogin": "sampaiodiego", + "contributors": [ + "rodrigok", + "sampaiodiego", + "web-flow" + ] + } + ] + }, + "3.14.0-rc.1": { + "node_version": "12.22.1", + "npm_version": "6.14.1", + "apps_engine_version": "1.25.0-alpha.4943", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [ + { + "pr": "21809", + "title": "Regression: Update fuselage for icons fix", + "userLogin": "MartinSchoeler", + "contributors": [ + "MartinSchoeler" + ] + }, + { + "pr": "21594", + "title": "[FIX] Too many request on loadHistory method", + "userLogin": "gabriellsh", + "contributors": [ + "gabriellsh", + "web-flow", + "ggazzo" + ] + }, + { + "pr": "21780", + "title": "regression: Markdown broken on safari", + "userLogin": "gabriellsh", + "contributors": [ + "gabriellsh", + "ggazzo", + "web-flow" + ] + }, + { + "pr": "21776", + "title": "Regression: Change CI files hashes for caching", + "userLogin": "sampaiodiego", + "contributors": [ + "sampaiodiego" + ] + }, + { + "pr": "21270", + "title": "[FIX] Discussions not showing in Safari", + "userLogin": "Kartik18g", + "milestone": "3.14.0", + "contributors": [ + "Kartik18g", + "web-flow", + "ggazzo" + ] + }, + { + "pr": "21741", + "title": "Regression: Reconnection not working properly due to changes on ECHD Proxy", + "userLogin": "rodrigok", + "description": "The ECHD Proxy implements a delay on websocket connection, the first implementation lost the reference to auto reconnect functionality.", + "milestone": "3.14.0", + "contributors": [ + "rodrigok" + ] + }, + { + "pr": "21731", + "title": "Regression: Fix scroll to bottom", + "userLogin": "ggazzo", + "contributors": [ + "ggazzo" + ] + }, + { + "pr": "21739", + "title": "[FIX] Toolbox icons order", + "userLogin": "tiagoevanp", + "contributors": [ + "tiagoevanp" + ] + }, + { + "pr": "21747", + "title": "Regression: Bold, italic and strike render (Original markdown)", + "userLogin": "gabriellsh", + "description": "Modified regex to avoid spaces between the marked text and the symbols. Also made it possible to apply the three markings at the same time, independing of order.", + "contributors": [ + "gabriellsh" + ] + }, + { + "pr": "21757", + "title": "Regression: Fix room not returning to the previous room after directory", + "userLogin": "ggazzo", + "contributors": [ + "ggazzo" + ] + }, + { + "pr": "21750", + "title": "Regression: Fix services Docker image build", + "userLogin": "sampaiodiego", + "milestone": "3.14.0", + "contributors": [ + "sampaiodiego" + ] + } + ] + }, + "3.14.0-rc.2": { + "node_version": "12.22.1", + "npm_version": "6.14.1", + "apps_engine_version": "1.25.0-alpha.4943", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [ + { + "pr": "21782", + "title": "[FIX] Omnichannel Activity Monitor closing chats returned to the queue", + "userLogin": "murtaza98", + "description": "Fix `VisitorInactivityMonitor` is still monitoring rooms that returned to `Queue Chats`", + "milestone": "3.14.0", + "contributors": [ + "murtaza98", + "web-flow", + "renatobecker" + ] + }, + { + "pr": "21812", + "title": "Regression: Problem with Importer's logs", + "userLogin": "pierre-lehnen-rc", + "milestone": "3.14.0", + "contributors": [ + "pierre-lehnen-rc", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21653", + "title": "Chore: Add tests for teams.update REST endpoint", + "userLogin": "g-thome", + "description": "add more tests to this endpoint", + "contributors": [ + "g-thome" + ] + }, + { + "pr": "21778", + "title": "QoL improvements to add channel to team flow", + "userLogin": "KevLehman", + "description": "- Fixed canAccessRoom validation\r\n- Added e2e tests\r\n- Removed channels that user cannot add to the team from autocomplete suggestions\r\n- Improved error messages", + "contributors": [ + "KevLehman", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21831", + "title": "Chore: Cache EE node_modules on CI", + "userLogin": "sampaiodiego", + "contributors": [ + "sampaiodiego" + ] + }, + { + "pr": "21768", + "title": "Regression: team sync not accepting multiple teams", + "userLogin": "pierre-lehnen-rc", + "milestone": "3.14.0", + "contributors": [ + "pierre-lehnen-rc", + "sampaiodiego", + "web-flow" + ] + }, + { + "pr": "21815", + "title": "regression: Italic being parsed with surrounding non-whitespace text", + "userLogin": "gabriellsh", + "contributors": [ + "gabriellsh", + "ggazzo", + "web-flow" + ] + }, + { + "pr": "21816", + "title": "Regression: Unread Threads Header and List", + "userLogin": "ggazzo", + "contributors": [ + "ggazzo", + "web-flow" + ] + }, + { + "pr": "21746", + "title": "[FIX] Attachment files are not rendered properly on SMS channels", + "userLogin": "renatobecker", + "milestone": "3.14.0", + "contributors": [ + "renatobecker", + "web-flow" + ] + } + ] + }, + "3.14.0-rc.3": { + "node_version": "12.22.1", + "npm_version": "6.14.1", + "apps_engine_version": "1.25.0", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [ + { + "pr": "21840", + "title": "Bump Apps-Engine version", + "userLogin": "d-gubert", + "milestone": "3.14.0", + "contributors": [ + "d-gubert" + ] + }, + { + "pr": "21839", + "title": "[FIX][Enterprise] Omnichannel simultaneous chat limit is not properly checking the limit by department", + "userLogin": "renatobecker", + "description": "The Omnichannel Concurrent Chat Limit feature is not working properly when checking the limit per department, the reason is that the algorithm that fetches the number of ongoing chats per agent wasn't considering the department of the subscriptions, hence, the number returned from DB was bigger than it should be.", + "milestone": "3.14.0", + "contributors": [ + "renatobecker", + "web-flow" + ] + }, + { + "pr": "21714", + "title": "Regression: Reactivate direct conversations only if all involved users are active", + "userLogin": "KevLehman", + "contributors": [ + "KevLehman", + "sampaiodiego", + "web-flow" + ] + } + ] + }, + "3.14.0-rc.4": { + "node_version": "12.22.1", + "npm_version": "6.14.1", + "apps_engine_version": "1.25.0", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [ + { + "pr": "21841", + "title": "bump fuselage", + "userLogin": "ggazzo", + "contributors": [ + "ggazzo", + "web-flow" + ] + }, + { + "pr": "21810", + "title": "[FIX] Duplicated header on admin's user contextualbar", + "userLogin": "dougfabris", + "description": "![image](https://user-images.githubusercontent.com/27704687/116125858-5ff60600-a69c-11eb-9859-41f7393b78bf.png)", + "contributors": [ + "dougfabris", + "gabriellsh", + "web-flow" + ] + } + ] + }, + "3.14.0": { + "node_version": "12.22.1", + "npm_version": "6.14.1", + "apps_engine_version": "1.25.0", + "mongo_versions": [ + "3.4", + "3.6", + "4.0" + ], + "pull_requests": [] } } } \ No newline at end of file diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 4e52167a816c..869b1e59a9e4 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -30,10 +30,10 @@ jobs: echo "github.event_name: ${{ github.event_name }}" cat $GITHUB_EVENT_PATH - - name: Use Node.js 12.21.0 + - name: Use Node.js 12.22.1 uses: actions/setup-node@v2 with: - node-version: "12.21.0" + node-version: "12.22.1" - uses: actions/checkout@v2 @@ -54,27 +54,29 @@ jobs: uses: actions/cache@v2 with: path: /home/runner/.cache/Cypress - key: ${{ runner.OS }}-cache-cypress-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }} + key: ${{ runner.OS }}-cache-cypress-${{ hashFiles('**/package-lock.json', '.github/workflows/build_and_test.yml') }} - name: Cache node modules if: steps.cache-cypress.outputs.cache-hit == 'true' id: cache-nodemodules uses: actions/cache@v2 with: - path: node_modules - key: ${{ runner.OS }}-node_modules-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }} + path: | + ./node_modules + ./ee/server/services/node_modules + key: ${{ runner.OS }}-node_modules-${{ hashFiles('**/package-lock.json', '.github/workflows/build_and_test.yml') }} - name: Cache meteor local uses: actions/cache@v2 with: path: ./.meteor/local - key: ${{ runner.OS }}-meteor_cache-${{ hashFiles('.meteor/versions') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }} + key: ${{ runner.OS }}-meteor_cache-${{ hashFiles('.meteor/versions', '.github/workflows/build_and_test.yml') }} - name: Cache meteor uses: actions/cache@v2 with: path: ~/.meteor - key: ${{ runner.OS }}-meteor-${{ hashFiles('.meteor/release') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }} + key: ${{ runner.OS }}-meteor-${{ hashFiles('.meteor/release', '.github/workflows/build_and_test.yml') }} - name: Install Meteor run: | @@ -108,6 +110,9 @@ jobs: if: steps.cache-nodemodules.outputs.cache-hit != 'true' || steps.cache-cypress.outputs.cache-hit != 'true' run: | meteor npm install + cd ./ee/server/services + npm install + cd - - run: meteor npm run lint @@ -146,7 +151,6 @@ jobs: - name: Try building micro services run: | cd ./ee/server/services - npm i npm run build rm -rf dist/ @@ -190,7 +194,7 @@ jobs: strategy: matrix: - node-version: ["12.21.0"] + node-version: ["12.22.1"] mongodb-version: ["3.4", "3.6", "4.0"] steps: @@ -232,15 +236,17 @@ jobs: uses: actions/cache@v2 with: path: /home/runner/.cache/Cypress - key: ${{ runner.OS }}-cache-cypress-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }} + key: ${{ runner.OS }}-cache-cypress-${{ hashFiles('**/package-lock.json', '.github/workflows/build_and_test.yml') }} - name: Cache node modules if: steps.cache-cypress.outputs.cache-hit == 'true' id: cache-nodemodules uses: actions/cache@v2 with: - path: node_modules - key: ${{ runner.OS }}-build-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('.github/workflows/build_and_test.yml') }} + path: | + ./node_modules + ./ee/server/services/node_modules + key: ${{ runner.OS }}-build-${{ hashFiles('**/package-lock.json', '.github/workflows/build_and_test.yml') }} - name: NPM install if: steps.cache-nodemodules.outputs.cache-hit != 'true' || steps.cache-cypress.outputs.cache-hit != 'true' @@ -545,10 +551,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Use Node.js 12.21.0 + - name: Use Node.js 12.22.1 uses: actions/setup-node@v2 with: - node-version: "12.21.0" + node-version: "12.22.1" - name: Parse for branch run: | diff --git a/.meteor/packages b/.meteor/packages index 02d1834c4f7b..f6f893dc5635 100644 --- a/.meteor/packages +++ b/.meteor/packages @@ -3,6 +3,7 @@ # 'meteor add' and 'meteor remove' will edit this file for you, # but you can also edit it by hand. +rocketchat:ddp rocketchat:mongo-config accounts-facebook@1.3.2 @@ -50,7 +51,6 @@ dispatch:run-as-user jalik:ufs@1.0.2 jalik:ufs-gridfs@1.0.2 jparker:gravatar -kadira:blaze-layout kadira:flow-router mizzao:timesync mrt:reactive-store diff --git a/.meteor/release b/.meteor/release index fd169bc4792f..b5323671fc44 100644 --- a/.meteor/release +++ b/.meteor/release @@ -1 +1 @@ -METEOR@2.1 +METEOR@2.1.1 diff --git a/.meteor/versions b/.meteor/versions index 5bcc27eebe93..069655e8be53 100644 --- a/.meteor/versions +++ b/.meteor/versions @@ -61,7 +61,6 @@ jparker:crypto-core@0.1.0 jparker:crypto-md5@0.1.1 jparker:gravatar@0.5.1 jquery@1.11.11 -kadira:blaze-layout@2.3.0 kadira:flow-router@2.12.1 konecty:multiple-instances-status@1.1.0 konecty:user-presence@2.6.3 @@ -116,6 +115,7 @@ reactive-dict@1.3.0 reactive-var@1.0.11 reload@1.3.1 retry@1.1.0 +rocketchat:ddp@0.0.1 rocketchat:i18n@0.0.1 rocketchat:livechat@0.0.1 rocketchat:mongo-config@0.0.1 diff --git a/.snapcraft/resources/prepareRocketChat b/.snapcraft/resources/prepareRocketChat index 99ba4dd41f59..fa31739a98d8 100755 --- a/.snapcraft/resources/prepareRocketChat +++ b/.snapcraft/resources/prepareRocketChat @@ -1,6 +1,6 @@ #!/bin/bash -curl -SLf "https://releases.rocket.chat/3.13.2/download/" -o rocket.chat.tgz +curl -SLf "https://releases.rocket.chat/3.14.0/download/" -o rocket.chat.tgz tar xf rocket.chat.tgz --strip 1 diff --git a/.snapcraft/resources/preparenode b/.snapcraft/resources/preparenode index 1a293dcdac3e..d4bdd8d4b186 100755 --- a/.snapcraft/resources/preparenode +++ b/.snapcraft/resources/preparenode @@ -1,6 +1,6 @@ #!/bin/bash -node_version="v12.21.0" +node_version="v12.22.1" unamem="$(uname -m)" if [[ $unamem == *aarch64* ]]; then diff --git a/.snapcraft/snap/snapcraft.yaml b/.snapcraft/snap/snapcraft.yaml index b72596d62445..a10085d412ce 100644 --- a/.snapcraft/snap/snapcraft.yaml +++ b/.snapcraft/snap/snapcraft.yaml @@ -7,7 +7,7 @@ # 5. `snapcraft snap` name: rocketchat-server -version: 3.13.2 +version: 3.14.0 summary: Rocket.Chat server description: Have your own Slack like online chat, built with Meteor. https://rocket.chat/ confinement: strict diff --git a/.storybook/main.js b/.storybook/main.js index ecfbbfab389f..a374e09f9453 100644 --- a/.storybook/main.js +++ b/.storybook/main.js @@ -1,3 +1,7 @@ +const path = require('path'); + +const webpack = require('webpack'); + module.exports = { stories: [ '../app/**/*.stories.js', @@ -6,5 +10,70 @@ module.exports = { ], addons: [ '@storybook/addon-essentials', + '@storybook/addon-postcss', ], + webpackFinal: async (config) => { + const cssRule = config.module.rules.find(({ test }) => test.test('index.css')); + + cssRule.use[2].options = { + ...cssRule.use[2].options, + postcssOptions: { + plugins: [ + require('postcss-custom-properties')({ preserve: true }), + require('postcss-media-minmax')(), + require('postcss-selector-not')(), + require('postcss-nested')(), + require('autoprefixer')(), + require('postcss-url')({ url: ({ absolutePath, relativePath, url }) => { + const absoluteDir = absolutePath.slice(0, -relativePath.length); + const relativeDir = path.relative(absoluteDir, path.resolve(__dirname, '../public')); + const newPath = path.join(relativeDir, url); + return newPath; + } }), + ], + }, + }; + + config.module.rules.push({ + test: /\.info$/, + type: 'json', + }); + + config.module.rules.push({ + test: /\.html$/, + use: '@settlin/spacebars-loader', + }); + + config.module.rules.push({ + test: /\.(ts|tsx)$/, + use: [ + { + loader: 'ts-loader', + options: { + compilerOptions: { + noEmit: false, + }, + }, + }, + ], + }); + + config.resolve.extensions.push('.ts', '.tsx'); + + config.plugins.push( + new webpack.NormalModuleReplacementPlugin( + /^meteor/, + require.resolve('./mocks/meteor.js'), + ), + new webpack.NormalModuleReplacementPlugin( + /(app)\/*.*\/(server)\/*/, + require.resolve('./mocks/empty.js'), + ), + ); + + config.mode = 'development'; + config.optimization.usedExports = true; + + return config; + }, }; diff --git a/.storybook/mocks/meteor.js b/.storybook/mocks/meteor.js index d224131c5e11..8117f1fe95a3 100644 --- a/.storybook/mocks/meteor.js +++ b/.storybook/mocks/meteor.js @@ -78,8 +78,6 @@ export const FlowRouter = { }), }; -export const BlazeLayout = {}; - export const Session = { get: () => {}, set: () => {}, diff --git a/.storybook/webpack.config.js b/.storybook/webpack.config.js deleted file mode 100644 index 65e19305d637..000000000000 --- a/.storybook/webpack.config.js +++ /dev/null @@ -1,65 +0,0 @@ -'use strict'; - -const path = require('path'); - -const webpack = require('webpack'); - -module.exports = async ({ config }) => { - const cssRule = config.module.rules.find(({ test }) => test.test('index.css')); - - cssRule.use[2].options.plugins = [ - require('postcss-custom-properties')({ preserve: true }), - require('postcss-media-minmax')(), - require('postcss-selector-not')(), - require('postcss-nested')(), - require('autoprefixer')(), - require('postcss-url')({ url: ({ absolutePath, relativePath, url }) => { - const absoluteDir = absolutePath.slice(0, -relativePath.length); - const relativeDir = path.relative(absoluteDir, path.resolve(__dirname, '../public')); - const newPath = path.join(relativeDir, url); - return newPath; - } }), - ]; - - config.module.rules.push({ - test: /\.info$/, - type: 'json', - }); - - config.module.rules.push({ - test: /\.html$/, - use: '@settlin/spacebars-loader', - }); - - config.module.rules.push({ - test: /\.(ts|tsx)$/, - use: [ - { - loader: 'ts-loader', - options: { - compilerOptions: { - noEmit: false, - }, - }, - }, - ], - }); - - config.resolve.extensions.push('.ts', '.tsx'); - - config.plugins.push( - new webpack.NormalModuleReplacementPlugin( - /^meteor/, - require.resolve('./mocks/meteor.js'), - ), - new webpack.NormalModuleReplacementPlugin( - /(app)\/*.*\/(server)\/*/, - require.resolve('./mocks/empty.js'), - ), - ); - - config.mode = 'development'; - config.optimization.usedExports = true; - - return config; -}; diff --git a/HISTORY.md b/HISTORY.md index bf35bc394b2b..592f38d8a11e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,12 +1,507 @@ +# 3.14.0 +`2021-04-27 · 9 🎉 · 9 🚀 · 55 🐛 · 38 🔍 · 30 👩‍💻👨‍💻` + +### Engine versions +- Node: `12.22.1` +- NPM: `6.14.1` +- MongoDB: `3.4, 3.6, 4.0` +- Apps-Engine: `1.25.0` + +### 🎉 New features + + +- **APPS:** Method to fetch Livechat Departments ([#21690](https://github.com/RocketChat/Rocket.Chat/pull/21690)) + + New method in the livechat bridge that allows apps to fetch departments that are enabled and have agents assigned + +- **APPS:** onInstall and onUninstall events ([#21565](https://github.com/RocketChat/Rocket.Chat/pull/21565)) + + Adding the `user` information when installing and uninstalling an App to the Apps-Engine. + +- **ENTERPRISE:** LDAP Teams Sync ([#21658](https://github.com/RocketChat/Rocket.Chat/pull/21658)) + +- **Enterprise:** Second layer encryption for data transport (alpha) ([#21692](https://github.com/RocketChat/Rocket.Chat/pull/21692)) + + The second layer encryption for data transport works implementing the ECDH algorithm where session keys are exchanged before the rest of the communication. This feature is **enterprise only** since it requires the micro-services architecture and it's in the early stage of tests as an **alpha** feature and documentation may not be available before the beta stage. + +- New set of rules for client code ([#21318](https://github.com/RocketChat/Rocket.Chat/pull/21318)) + + This _small_ PR does the following: + + - Now **React** is the web client's first-class citizen, being **loaded before Blaze**. Thus, `BlazeLayout` calls render templates inside of a React component (`BlazeLayoutWrapper`); + - Main client startup code, including polyfills, is written in **TypeScript**; + - At the moment, routes are treated as regular startup code; it's expected that `FlowRouter` will be deprecated in favor of a new routing library; + - **React** was updated to major version **17**, deprecating the usage of `React` as namespace (e.g. use `memo()` instead of `React.memo()`); + - The `client/` and `ee/client/` directory are linted with a **custom ESLint configuration** that includes: + - **Prettier**; + - `react-hooks/*` rules for TypeScript files; + - `react/no-multi-comp`, enforcing the rule of **one single React component per module**; + - `react/display-name`, which enforces that **React components must have a name for debugging**; + - `import/named`, avoiding broken named imports. + - A bunch of components were refactored to match the new ESLint rules. + +- On Hold system messages ([#21360](https://github.com/RocketChat/Rocket.Chat/pull/21360)) + + ![image](https://user-images.githubusercontent.com/34130764/115442079-3a49a680-a22f-11eb-9ee8-6c705097cd57.png) + +- Password history ([#21607](https://github.com/RocketChat/Rocket.Chat/pull/21607)) + + - Store each user's previously used passwords in a `passwordHistory` field (in the `users` record); + - Users' previously used passwords are stored in their `passwordHistory` even when the setting is disabled; + - Add "Password History" setting -- when enabled, it blocks users from reusing their most recent passwords; + - Convert `comparePassword` file to TypeScript. + + ![Password_Change](https://user-images.githubusercontent.com/36537004/115035168-ac726200-9ea2-11eb-93c6-fc8182ba5f3f.png) + ![Password_History](https://user-images.githubusercontent.com/36537004/115035175-ad0af880-9ea2-11eb-9f40-94c6327a9854.png) + +- REST endpoint `teams.update` ([#21134](https://github.com/RocketChat/Rocket.Chat/pull/21134)) + + add teams.update endpoint + +- Standard Importer Structure ([#18357](https://github.com/RocketChat/Rocket.Chat/pull/18357)) + +### 🚀 Improvements + + +- **APPS:** Scheduler option to skip immediate execution of recurring jobs ([#21353](https://github.com/RocketChat/Rocket.Chat/pull/21353)) + + Create and schedule a task manually at `scheduleRecurring` method so the first iteration runs after the configured interval. This is accomplished by adding the setting `skipImmediate: true` when setting up the task. + +- Add error messages to the creation of channels or usernames containing reserved words ([#21016](https://github.com/RocketChat/Rocket.Chat/pull/21016)) + + Display error messages when the user attempts to create or edit users' or channels' names with any of the following words (**case-insensitive**): + - admin; + - administrator; + - system; + - user. + ![create-channel](https://user-images.githubusercontent.com/36537004/110132223-b421ef80-7da9-11eb-82bc-f0d4e1df967f.png) + ![register-username](https://user-images.githubusercontent.com/36537004/110132234-b71ce000-7da9-11eb-904e-580233625951.png) + ![change-channel](https://user-images.githubusercontent.com/36537004/110143057-96f31e00-7db5-11eb-994a-39ae9e63392e.png) + ![change-username](https://user-images.githubusercontent.com/36537004/110143065-98244b00-7db5-11eb-9d13-afc5dc9866de.png) + +- add permission check when adding a channel to a team ([#21689](https://github.com/RocketChat/Rocket.Chat/pull/21689)) + + add permission check for each room + +- Add proxy for data export ([#20998](https://github.com/RocketChat/Rocket.Chat/pull/20998)) + + Add a proxy for data export downloads (instead of just linking ufs urls) so we can have more control over its response. Also added a human readable message when the user tries to download the user-data unauthenticated. + +- Add support to range downloads on file system storage ([#21463](https://github.com/RocketChat/Rocket.Chat/pull/21463)) + +- Alert on team deletion ([#21617](https://github.com/RocketChat/Rocket.Chat/pull/21617)) + + Screen Shot 2021-04-16 at 7 03 30 PM + +- Do not require pre-configured tags in Omnichannel chats ([#21488](https://github.com/RocketChat/Rocket.Chat/pull/21488)) + +- OEmbed details by requesting using the accept language header on the request ([#21686](https://github.com/RocketChat/Rocket.Chat/pull/21686)) + + - Send `Accept-Language` header on oembed requests + +- Resize custom emojis on upload instead of saving at max res ([#21593](https://github.com/RocketChat/Rocket.Chat/pull/21593)) + + - Create new MediaService (ideally, should be in charge of all media-related operations) + - Resize emojis to 128x128 + +### 🐛 Bug fixes + + +- **Enterprise:** Omnichannel simultaneous chat limit is not properly checking the limit by department ([#21839](https://github.com/RocketChat/Rocket.Chat/pull/21839)) + + The Omnichannel Concurrent Chat Limit feature is not working properly when checking the limit per department, the reason is that the algorithm that fetches the number of ongoing chats per agent wasn't considering the department of the subscriptions, hence, the number returned from DB was bigger than it should be. + +- Add tag input to Closing Chat modal ([#21462](https://github.com/RocketChat/Rocket.Chat/pull/21462)) + +- Admin Users list pagination ([#21469](https://github.com/RocketChat/Rocket.Chat/pull/21469)) + + - Fix Administration/Users pagination + +- Allow deletion of own account for passwordless accounts (e.g. OAUTH) ([#21119](https://github.com/RocketChat/Rocket.Chat/pull/21119) by [@wolbernd](https://github.com/wolbernd)) + +- Allows more than 25 discussions/files to be loaded in the contextualbar ([#21511](https://github.com/RocketChat/Rocket.Chat/pull/21511) by [@Jeanstaquet](https://github.com/Jeanstaquet)) + + In some places, you could not load more than 25 threads/discussions/files on the screen when searching the lists in the contextualbar. + Threads & list are numbered for a better view of the solution + + + https://user-images.githubusercontent.com/45966964/114222225-93335800-996e-11eb-833f-568e83129aae.mp4 + +- Allows more than 25 threads to be loaded, fixes #21507 ([#21508](https://github.com/RocketChat/Rocket.Chat/pull/21508) by [@Jeanstaquet](https://github.com/Jeanstaquet)) + +- Allows to display more than 25 users maximum in the users list ([#21518](https://github.com/RocketChat/Rocket.Chat/pull/21518) by [@Jeanstaquet](https://github.com/Jeanstaquet)) + + Now when you scroll to the bottom of the users list, it shows more users. Before the fix, the limit for the query for loadMore was calculated so that no additional users could be loaded. + + Before + + https://user-images.githubusercontent.com/45966964/114249739-baece500-999b-11eb-9bb0-3a5bcee18ad8.mp4 + + After + + + https://user-images.githubusercontent.com/45966964/114249895-364e9680-999c-11eb-985c-47aedc763488.mp4 + +- App installation from marketplace not correctly displaying the permissions ([#21470](https://github.com/RocketChat/Rocket.Chat/pull/21470)) + + Fixes the marketplace app installation not correctly displaying the permissions modal. + +- Archive permissions for room moderator ([#21563](https://github.com/RocketChat/Rocket.Chat/pull/21563)) + +- Attachment files are not rendered properly on SMS channels ([#21746](https://github.com/RocketChat/Rocket.Chat/pull/21746)) + +- Audio message same pattern as image message ([#21466](https://github.com/RocketChat/Rocket.Chat/pull/21466)) + + ![image](https://user-images.githubusercontent.com/17487063/113760168-4c363000-96ec-11eb-9138-0fbcedb3fa42.png) + +- Avoid sidebar being broke ([#21490](https://github.com/RocketChat/Rocket.Chat/pull/21490)) + +- Change margin size for quote messages ([#21461](https://github.com/RocketChat/Rocket.Chat/pull/21461)) + + ![image](https://user-images.githubusercontent.com/17487063/113723723-02d3e980-96c8-11eb-9bc7-70aab5ea8091.png) + +- Change team private info text ([#21535](https://github.com/RocketChat/Rocket.Chat/pull/21535)) + +- Change the active appearance for toolbox buttons ([#21416](https://github.com/RocketChat/Rocket.Chat/pull/21416)) + + ![image](https://user-images.githubusercontent.com/17487063/113359447-2d1b5500-931e-11eb-81fa-86f60fcee3a9.png) + +- Checking 'start-discussion' Permission for MessageBox Actions ([#21564](https://github.com/RocketChat/Rocket.Chat/pull/21564) by [@yash-rajpal](https://github.com/yash-rajpal)) + + Permissions 'start-discussion-other-user' and 'start-discussion' are checked everywhere before letting anyone start any discussions, this permission check was missing for message box actions, so added it. + +- Close chat button is not available for Omnichannel agents ([#21481](https://github.com/RocketChat/Rocket.Chat/pull/21481)) + +- Correcting the case there are no result in admin users list ([#21556](https://github.com/RocketChat/Rocket.Chat/pull/21556) by [@Jeanstaquet](https://github.com/Jeanstaquet)) + + I added a default case to the total when there are no result to the user's query + +- Discussions not showing in Safari ([#21270](https://github.com/RocketChat/Rocket.Chat/pull/21270) by [@Kartik18g](https://github.com/Kartik18g)) + +- Don't allow whitespace on bold, italic and strike ([#21483](https://github.com/RocketChat/Rocket.Chat/pull/21483)) + + Stops the original markdown rendered from rendering empty bold, italic and strike text. Stops `_ _`, `* *` and `~ ~` + +- Don't ask again modals blinking ([#21454](https://github.com/RocketChat/Rocket.Chat/pull/21454)) + + Made the check before opening the modal. + +- Duplicated header on admin's user contextualbar ([#21810](https://github.com/RocketChat/Rocket.Chat/pull/21810)) + + ![image](https://user-images.githubusercontent.com/27704687/116125858-5ff60600-a69c-11eb-9859-41f7393b78bf.png) + +- Error when editing Omnichannel rooms without custom fields ([#21450](https://github.com/RocketChat/Rocket.Chat/pull/21450)) + +- Fix the bugs opening discussions ([#21557](https://github.com/RocketChat/Rocket.Chat/pull/21557) by [@Jeanstaquet](https://github.com/Jeanstaquet)) + + I added the right row export to display the discussions list + +- Generic Attachment broken somehow ([#21657](https://github.com/RocketChat/Rocket.Chat/pull/21657)) + +- Header component breaking if user is not part of teams room. ([#21465](https://github.com/RocketChat/Rocket.Chat/pull/21465)) + +- Livechat not retrieving messages ([#21644](https://github.com/RocketChat/Rocket.Chat/pull/21644) by [@cuonghuunguyen](https://github.com/cuonghuunguyen)) + +- Make Omnichannel's closing chat button the last action in the toolbox ([#21476](https://github.com/RocketChat/Rocket.Chat/pull/21476)) + +- Margins on contextual bar information ([#21457](https://github.com/RocketChat/Rocket.Chat/pull/21457)) + + ### Room + **Before** + ![image](https://user-images.githubusercontent.com/27704687/115080812-ba8fa500-9ed9-11eb-9078-3625603bf92b.png) + + **After** + ![image](https://user-images.githubusercontent.com/27704687/115080966-e9a61680-9ed9-11eb-929f-6516c1563e99.png) + + ### Livechat + ![image](https://user-images.githubusercontent.com/27704687/113640101-1859fc80-9651-11eb-88f8-09a899953988.png) + +- Message Block ordering ([#21464](https://github.com/RocketChat/Rocket.Chat/pull/21464)) + + Reactions should come before reply button. + ![image](https://user-images.githubusercontent.com/40830821/113748926-6f0e1780-96df-11eb-93a5-ddcfa891413e.png) + +- Message link null corrupts message rendering ([#21579](https://github.com/RocketChat/Rocket.Chat/pull/21579)) + + Additional checks on message_link field before rendering message contents + +- Omnichannel Activity Monitor closing chats returned to the queue ([#21782](https://github.com/RocketChat/Rocket.Chat/pull/21782)) + + Fix `VisitorInactivityMonitor` is still monitoring rooms that returned to `Queue Chats` + +- Omnichannel current chats and agents grid aren't sorting by status properly ([#21616](https://github.com/RocketChat/Rocket.Chat/pull/21616)) + +- Omnichannel queue manager returning outdated room object ([#21485](https://github.com/RocketChat/Rocket.Chat/pull/21485)) + + The Omnichannel Queue Manager is returning outdated room object when delegating the chat to an agent, hence, our Livechat widget is affected and the agent assigned to the chat is not displayed on the widget, only after refreshing/reloading. + +- Omnichannel room information panel breaking due to lack of data verification ([#21608](https://github.com/RocketChat/Rocket.Chat/pull/21608)) + +- public teams not appearing on spotlight search results ([#21495](https://github.com/RocketChat/Rocket.Chat/pull/21495)) + +- Remove all agent subscriptions when an Omnichannel chat is closed ([#21509](https://github.com/RocketChat/Rocket.Chat/pull/21509)) + +- Remove size prop from StatusBullet component ([#21428](https://github.com/RocketChat/Rocket.Chat/pull/21428)) + +- Rename Omnichannel Rooms, Inquiries and Subscriptions when the Contact Name changes ([#21513](https://github.com/RocketChat/Rocket.Chat/pull/21513)) + +- Rename team not working properly ([#21552](https://github.com/RocketChat/Rocket.Chat/pull/21552)) + +- Selected channels are not showing in Teams ([#21669](https://github.com/RocketChat/Rocket.Chat/pull/21669) by [@sumukhah](https://github.com/sumukhah)) + +- Send alternative color to unread sidebar icon ([#21432](https://github.com/RocketChat/Rocket.Chat/pull/21432)) + + ![image](https://user-images.githubusercontent.com/17487063/113469819-08f76b00-9427-11eb-942e-783c186ba7cd.png) + +- Show direct rooms as readonly when one of the users is deactivated ([#21684](https://github.com/RocketChat/Rocket.Chat/pull/21684)) + +- Tag component is no longer rendering on Chat Room Information panel ([#21429](https://github.com/RocketChat/Rocket.Chat/pull/21429)) + +- Team types in admin -> rooms. ([#21612](https://github.com/RocketChat/Rocket.Chat/pull/21612)) + + ![print](https://user-images.githubusercontent.com/40830821/115068327-82339b00-9ec8-11eb-8e37-726baf9d2db0.jpg) + +- Team's channels list for teams with too many channels ([#21491](https://github.com/RocketChat/Rocket.Chat/pull/21491)) + + - Fix teams.listRooms pagination for non-admin users + +- Too many request on loadHistory method ([#21594](https://github.com/RocketChat/Rocket.Chat/pull/21594)) + +- Toolbox icons order ([#21739](https://github.com/RocketChat/Rocket.Chat/pull/21739)) + +- Typos/missing elements in the French translation ([#21525](https://github.com/RocketChat/Rocket.Chat/pull/21525) by [@Jeanstaquet](https://github.com/Jeanstaquet)) + + - I have corrected some typos in the translation + - I added a translation for missing words + - I took the opportunity to correct a mistranslated word + - Test_Desktop_Notifications was missing in the EN and FR file + ![image](https://user-images.githubusercontent.com/45966964/114290186-e7792d80-9a7d-11eb-8164-3b5e72e93703.png) + +- Updating a message causing URLs to be parsed even within markdown code ([#21489](https://github.com/RocketChat/Rocket.Chat/pull/21489)) + + - Fix `updateMessage` to avoid parsing URLs inside markdown + - Honor `parseUrls` property when updating messages + +- Use async await in TeamChannels delete channel action ([#21534](https://github.com/RocketChat/Rocket.Chat/pull/21534)) + +- User status out of sync ([#21656](https://github.com/RocketChat/Rocket.Chat/pull/21656)) + +- Wrong title on Omnichannel contact information panel ([#21682](https://github.com/RocketChat/Rocket.Chat/pull/21682)) + +- Wrong useMemo on Priorities EE field. ([#21453](https://github.com/RocketChat/Rocket.Chat/pull/21453)) + +- Wrong user in user info ([#21451](https://github.com/RocketChat/Rocket.Chat/pull/21451)) + + Fixed some race conditions in admin. + + Self DMs used to be created with the userId duplicated. Sometimes rooms can have 2 equal uids, but it's a self DM. Fixed a getter so this isn't a problem anymore. + +
+🔍 Minor changes + + +- Doc: Corrected links to documentation of rocket.chat README.md ([#20478](https://github.com/RocketChat/Rocket.Chat/pull/20478) by [@joshi008](https://github.com/joshi008)) + + The link for documentation in the readme was previously https://rocket.chat/docs/ while that was not working and according to the website it was https://docs.rocket.chat/ + The link for deployment methods in readme was corrected from https://rocket.chat/docs/installation/paas-deployments/ to https://docs.rocket.chat/installation/paas-deployments + Some more links to the documentations were giving 404 error which hence updated. + +- [Improve] Remove useless tabbar options from Omnichannel rooms ([#21561](https://github.com/RocketChat/Rocket.Chat/pull/21561)) + +- A React-based replacement for BlazeLayout ([#21527](https://github.com/RocketChat/Rocket.Chat/pull/21527)) + + - The Meteor package **`kadira:blaze-layout` was removed**; + - A **global subscription** for the current application layout (**`appLayout`**) replaces `BlazeLayout` entirely; + - The **`#react-root` element** is rendered on server-side instead of dynamically injected into the DOM tree; + - The **"page loading" throbber** is now rendered on the React tree; + - The **`renderRouteComponent` helper was removed**; + - Some code run without any criteria on **`main` template** module was moved into **client startup modules**; + - React portals used to embed Blaze templates have their own subscription (**`blazePortals`**); + - Some **route components were refactored** to remove a URL path trap originally disabled by `renderRouteComponent`; + - A new component to embed the DOM nodes generated by **`RoomManager`** was created. + +- Add ')' after Date and Time in DB migration ([#21519](https://github.com/RocketChat/Rocket.Chat/pull/21519) by [@im-adithya](https://github.com/im-adithya)) + +- Bump Apps-Engine version ([#21840](https://github.com/RocketChat/Rocket.Chat/pull/21840)) + +- bump fuselage ([#21841](https://github.com/RocketChat/Rocket.Chat/pull/21841)) + +- Bump Livechat Version ([#21694](https://github.com/RocketChat/Rocket.Chat/pull/21694)) + +- Chore: Add tests for teams.update REST endpoint ([#21653](https://github.com/RocketChat/Rocket.Chat/pull/21653)) + + add more tests to this endpoint + +- Chore: Cache EE node_modules on CI ([#21831](https://github.com/RocketChat/Rocket.Chat/pull/21831)) + +- Chore: Do not stop animations on Test Mode ([#21484](https://github.com/RocketChat/Rocket.Chat/pull/21484)) + +- Chore: Increase testing coverage on password policy class ([#21482](https://github.com/RocketChat/Rocket.Chat/pull/21482)) + +- Chore: Meteor update to 2.1.1 ([#21494](https://github.com/RocketChat/Rocket.Chat/pull/21494)) + + Basically Node update to version 12.22.1 + + Meteor change log https://github.com/meteor/meteor/blob/devel/History.md#v211-2021-04-06 + +- Chore: Remove control character from room model operation ([#21493](https://github.com/RocketChat/Rocket.Chat/pull/21493)) + +- Fix typo in app/apps/README file ([#21204](https://github.com/RocketChat/Rocket.Chat/pull/21204) by [@sauravjoshi23](https://github.com/sauravjoshi23)) + +- Fix: Missing module `eventemitter3` for micro services ([#21611](https://github.com/RocketChat/Rocket.Chat/pull/21611)) + + - Fix error when running micro services after version 3.12 + - Fix build of docker image version latest for micro services + +- Language update from LingoHub 🤖 on 2021-04-05Z ([#21446](https://github.com/RocketChat/Rocket.Chat/pull/21446)) + +- Language update from LingoHub 🤖 on 2021-04-12Z ([#21530](https://github.com/RocketChat/Rocket.Chat/pull/21530)) + +- Language update from LingoHub 🤖 on 2021-04-19Z ([#21642](https://github.com/RocketChat/Rocket.Chat/pull/21642)) + +- Merge master into develop & Set version to 3.14.0-develop ([#21441](https://github.com/RocketChat/Rocket.Chat/pull/21441)) + +- QoL improvements to add channel to team flow ([#21778](https://github.com/RocketChat/Rocket.Chat/pull/21778)) + + - Fixed canAccessRoom validation + - Added e2e tests + - Removed channels that user cannot add to the team from autocomplete suggestions + - Improved error messages + +- Regression: Bold, italic and strike render (Original markdown) ([#21747](https://github.com/RocketChat/Rocket.Chat/pull/21747)) + + Modified regex to avoid spaces between the marked text and the symbols. Also made it possible to apply the three markings at the same time, independing of order. + +- regression: Cannot enable e2e in direct room. ([#21650](https://github.com/RocketChat/Rocket.Chat/pull/21650)) + +- Regression: Change CI files hashes for caching ([#21776](https://github.com/RocketChat/Rocket.Chat/pull/21776)) + +- Regression: Edit user in admin breaking ([#21613](https://github.com/RocketChat/Rocket.Chat/pull/21613)) + +- Regression: Fix room not returning to the previous room after directory ([#21757](https://github.com/RocketChat/Rocket.Chat/pull/21757)) + +- Regression: Fix scroll to bottom ([#21731](https://github.com/RocketChat/Rocket.Chat/pull/21731)) + +- Regression: Fix services Docker image build ([#21750](https://github.com/RocketChat/Rocket.Chat/pull/21750)) + +- regression: Italic being parsed with surrounding non-whitespace text ([#21815](https://github.com/RocketChat/Rocket.Chat/pull/21815)) + +- Regression: Legacy Banner Position ([#21598](https://github.com/RocketChat/Rocket.Chat/pull/21598)) + + ### Before: + ![image](https://user-images.githubusercontent.com/27704687/114961773-dc3c4e00-9e3f-11eb-9a32-e882db3fbfbc.png) + + ### After + ![image](https://user-images.githubusercontent.com/27704687/114961673-a6976500-9e3f-11eb-9238-a12870d7db8f.png) + +- regression: Markdown broken on safari ([#21780](https://github.com/RocketChat/Rocket.Chat/pull/21780)) + +- Regression: Problem with Importer's logs ([#21812](https://github.com/RocketChat/Rocket.Chat/pull/21812)) + +- Regression: React + Blaze reconciliation ([#21567](https://github.com/RocketChat/Rocket.Chat/pull/21567)) + +- Regression: Reactivate direct conversations only if all involved users are active ([#21714](https://github.com/RocketChat/Rocket.Chat/pull/21714)) + +- Regression: Reconnection not working properly due to changes on ECHD Proxy ([#21741](https://github.com/RocketChat/Rocket.Chat/pull/21741)) + + The ECHD Proxy implements a delay on websocket connection, the first implementation lost the reference to auto reconnect functionality. + +- regression: Team Channels actions ([#21417](https://github.com/RocketChat/Rocket.Chat/pull/21417)) + +- Regression: team sync not accepting multiple teams ([#21768](https://github.com/RocketChat/Rocket.Chat/pull/21768)) + +- Regression: Unread Threads Header and List ([#21816](https://github.com/RocketChat/Rocket.Chat/pull/21816)) + +- Regression: Update fuselage for icons fix ([#21809](https://github.com/RocketChat/Rocket.Chat/pull/21809)) + +
+ +### 👩‍💻👨‍💻 Contributors 😍 + +- [@Jeanstaquet](https://github.com/Jeanstaquet) +- [@Kartik18g](https://github.com/Kartik18g) +- [@cuonghuunguyen](https://github.com/cuonghuunguyen) +- [@im-adithya](https://github.com/im-adithya) +- [@joshi008](https://github.com/joshi008) +- [@sauravjoshi23](https://github.com/sauravjoshi23) +- [@sumukhah](https://github.com/sumukhah) +- [@wolbernd](https://github.com/wolbernd) +- [@yash-rajpal](https://github.com/yash-rajpal) + +### 👩‍💻👨‍💻 Core Team 🤓 + +- [@KevLehman](https://github.com/KevLehman) +- [@MartinSchoeler](https://github.com/MartinSchoeler) +- [@d-gubert](https://github.com/d-gubert) +- [@dougfabris](https://github.com/dougfabris) +- [@g-thome](https://github.com/g-thome) +- [@gabriellsh](https://github.com/gabriellsh) +- [@ggazzo](https://github.com/ggazzo) +- [@graywolf336](https://github.com/graywolf336) +- [@lolimay](https://github.com/lolimay) +- [@lucassartor](https://github.com/lucassartor) +- [@matheusbsilva137](https://github.com/matheusbsilva137) +- [@murtaza98](https://github.com/murtaza98) +- [@pierre-lehnen-rc](https://github.com/pierre-lehnen-rc) +- [@r0zbot](https://github.com/r0zbot) +- [@rafaelblink](https://github.com/rafaelblink) +- [@renatobecker](https://github.com/renatobecker) +- [@rodrigok](https://github.com/rodrigok) +- [@sampaiodiego](https://github.com/sampaiodiego) +- [@tassoevan](https://github.com/tassoevan) +- [@thassiov](https://github.com/thassiov) +- [@tiagoevanp](https://github.com/tiagoevanp) + +# 3.13.3 +`2021-04-20 · 2 🐛 · 3 👩‍💻👨‍💻` + +### Engine versions +- Node: `12.21.0` +- NPM: `6.14.8` +- MongoDB: `3.4, 3.6, 4.0` +- Apps-Engine: `1.24.1` + +### 🐛 Bug fixes + + +- Livechat not retrieving messages ([#21644](https://github.com/RocketChat/Rocket.Chat/pull/21644) by [@cuonghuunguyen](https://github.com/cuonghuunguyen)) + +- Team's channels list for teams with too many channels ([#21491](https://github.com/RocketChat/Rocket.Chat/pull/21491)) + + - Fix teams.listRooms pagination for non-admin users + +### 👩‍💻👨‍💻 Contributors 😍 + +- [@cuonghuunguyen](https://github.com/cuonghuunguyen) + +### 👩‍💻👨‍💻 Core Team 🤓 + +- [@KevLehman](https://github.com/KevLehman) +- [@sampaiodiego](https://github.com/sampaiodiego) + # 3.13.2 -`2021-04-14 · 1 🐛 · 3 👩‍💻👨‍💻` +`2021-04-14 · 1 🐛 · 1 🔍 · 3 👩‍💻👨‍💻` + +### Engine versions +- Node: `12.21.0` +- NPM: `6.14.8` +- MongoDB: `3.4, 3.6, 4.0` +- Apps-Engine: `1.24.1` ### 🐛 Bug fixes - Security Hotfix (https://docs.rocket.chat/guides/security/security-updates) +
+🔍 Minor changes + + +- Release 3.13.2 ([#21570](https://github.com/RocketChat/Rocket.Chat/pull/21570)) + +
+ ### 👩‍💻👨‍💻 Core Team 🤓 - [@KevLehman](https://github.com/KevLehman) @@ -648,6 +1143,24 @@ - [@tassoevan](https://github.com/tassoevan) - [@tiagoevanp](https://github.com/tiagoevanp) +# 3.12.5 +`2021-04-20 · 1 🐛 · 1 👩‍💻👨‍💻` + +### Engine versions +- Node: `12.18.4` +- NPM: `6.14.8` +- MongoDB: `3.4, 3.6, 4.0` +- Apps-Engine: `1.23.0` + +### 🐛 Bug fixes + + +- Livechat not retrieving messages ([#21644](https://github.com/RocketChat/Rocket.Chat/pull/21644) by [@cuonghuunguyen](https://github.com/cuonghuunguyen)) + +### 👩‍💻👨‍💻 Contributors 😍 + +- [@cuonghuunguyen](https://github.com/cuonghuunguyen) + # 3.12.2 `2021-03-26 · 2 🐛 · 4 👩‍💻👨‍💻` @@ -1246,6 +1759,24 @@ - [@tassoevan](https://github.com/tassoevan) - [@tiagoevanp](https://github.com/tiagoevanp) +# 3.11.5 +`2021-04-20 · 1 🐛 · 1 👩‍💻👨‍💻` + +### Engine versions +- Node: `12.18.4` +- NPM: `6.14.8` +- MongoDB: `3.4, 3.6, 4.0` +- Apps-Engine: `1.22.2` + +### 🐛 Bug fixes + + +- Livechat not retrieving messages ([#21644](https://github.com/RocketChat/Rocket.Chat/pull/21644) by [@cuonghuunguyen](https://github.com/cuonghuunguyen)) + +### 👩‍💻👨‍💻 Contributors 😍 + +- [@cuonghuunguyen](https://github.com/cuonghuunguyen) + # 3.11.2 `2021-02-28 · 3 🐛 · 3 👩‍💻👨‍💻` diff --git a/README.md b/README.md index 3af520c6f658..70df6ab511ec 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ Installing snaps is very quick. By running that command you have your full Rocke Our snap features a built-in reverse proxy that can request and maintain free Let's Encrypt SSL certificates. You can go from zero to a public-facing SSL-secured Rocket.Chat server in less than 5 minutes. -Find out more information about our snaps [here](https://rocket.chat/docs/installation/manual-installation/ubuntu/snaps/). +Find out more information about our snaps [here](https://docs.rocket.chat/installation/snaps). ## DigitalOcean droplet @@ -156,7 +156,7 @@ Host your docker container at [sloppy.io](http://sloppy.io). Get an account and ## Docker -[Deploy with docker compose](https://rocket.chat/docs/installation/docker-containers/docker-compose/) +[Deploy with docker compose](https://docs.rocket.chat/installation/docker-containers#3-installing-docker-and-docker-compose) [![Rocket.Chat logo](https://d207aa93qlcgug.cloudfront.net/1.95.5.qa/img/nav/docker-logo-loggedout.png)](https://hub.docker.com/r/rocketchat/rocket.chat/) @@ -194,7 +194,7 @@ Add Rocket.Chat to this world famous time tested small enterprise server today. [![Koozali SME](https://raw.githubusercontent.com/Sing-Li/bbug/master/images/koozali.png)](https://wiki.contribs.org/Rocket_Chat) ## Ubuntu VPS -Follow these [deployment instructions](https://rocket.chat/docs/installation/manual-installation/ubuntu/). +Follow these [deployment instructions](https://docs.rocket.chat/installation/manual-installation/ubuntu). ## D2C.io Deploy Rocket.Chat stack to your server with [D2C](https://d2c.io/). Scale with a single click, check live logs and metrics: @@ -341,7 +341,7 @@ We are developing the APIs based on the competition, so stay tuned and you will ## Documentation -Check out [Rocket.Chat documentation](https://rocket.chat/docs/). +Check out [Rocket.Chat documentation](https://docs.rocket.chat/). ## License @@ -374,11 +374,11 @@ meteor debug ``` You'll find a nodejs icon in the developer console. -If you are not a developer and just want to run the server - see [deployment methods](https://rocket.chat/docs/installation/paas-deployments/). +If you are not a developer and just want to run the server - see [deployment methods](https://docs.rocket.chat/installation/paas-deployments). ## Branching Model -See [Branches and Releases](https://rocket.chat/docs/developer-guides/branches-and-releases/). +See [Branches and Releases](https://docs.rocket.chat/guides/developer/branches-and-releases). It is based on [Gitflow Workflow](http://nvie.com/posts/a-successful-git-branching-model/), reference section below is derived from Vincent Driessen at nvie. @@ -390,7 +390,7 @@ If you want to help, send an email to support at rocket.chat to be invited to th ## How to Contribute -Already a JavaScript developer? Familiar with Meteor? [Pick an issue](https://github.com/RocketChat/Rocket.Chat/labels/contrib%3A%20easy), push a PR and instantly become a member of Rocket.Chat's international contributors' community. For more information, check out our [Contributing Guide](.github/CONTRIBUTING.md) and our [Official Documentation for Contributors](https://rocket.chat/docs/contributing/). +Already a JavaScript developer? Familiar with Meteor? [Pick an issue](https://github.com/RocketChat/Rocket.Chat/labels/contrib%3A%20easy), push a PR and instantly become a member of Rocket.Chat's international contributors' community. For more information, check out our [Contributing Guide](.github/CONTRIBUTING.md) and our [Official Documentation for Contributors](https://docs.rocket.chat/contributors/contributing). A lot of work has already gone into Rocket.Chat, but we have much bigger plans for it! diff --git a/app/analytics/client/loadScript.js b/app/analytics/client/loadScript.js index 7b539cd222d9..491f340e734b 100644 --- a/app/analytics/client/loadScript.js +++ b/app/analytics/client/loadScript.js @@ -1,12 +1,11 @@ import { Meteor } from 'meteor/meteor'; -import { Tracker } from 'meteor/tracker'; import { Template } from 'meteor/templating'; import { settings } from '../../settings'; import { hex_sha1 } from '../../utils'; -Template.body.onRendered(() => { - Tracker.autorun((c) => { +Template.body.onRendered(function() { + this.autorun((c) => { const piwikUrl = settings.get('PiwikAnalytics_enabled') && settings.get('PiwikAnalytics_url'); const piwikSiteId = piwikUrl && settings.get('PiwikAnalytics_siteId'); const piwikPrependDomain = piwikUrl && settings.get('PiwikAnalytics_prependDomain'); diff --git a/app/api/server/default/info.js b/app/api/server/default/info.js index 7c397de09cb1..861d9dfb36ac 100644 --- a/app/api/server/default/info.js +++ b/app/api/server/default/info.js @@ -17,3 +17,15 @@ API.default.addRoute('info', { authRequired: false }, { }); }, }); + +API.default.addRoute('ecdh_proxy/initEncryptedSession', { authRequired: false }, { + post() { + return { + statusCode: 200, + body: { + success: false, + error: 'Not Acceptable', + }, + }; + }, +}); diff --git a/app/api/server/lib/rooms.js b/app/api/server/lib/rooms.js index 9bf407ae2396..99aef384ed77 100644 --- a/app/api/server/lib/rooms.js +++ b/app/api/server/lib/rooms.js @@ -136,11 +136,11 @@ export async function findRoomsAvailableForTeams({ uid, name }) { }, }; - const userRooms = Subscriptions.findByUserIdAndType(uid, 'p', { fields: { rid: 1 } }) + const userRooms = Subscriptions.findByUserIdAndRoles(uid, ['owner'], { fields: { rid: 1 } }) .fetch() .map((item) => item.rid); - const rooms = await Rooms.findChannelAndGroupListWithoutTeamsByNameStarting(name, userRooms, options).toArray(); + const rooms = await Rooms.findChannelAndGroupListWithoutTeamsByNameStartingByOwner(uid, name, userRooms, options).toArray(); return { items: rooms, diff --git a/app/api/server/v1/emoji-custom.js b/app/api/server/v1/emoji-custom.js index 249331a3e832..d5f489b9798e 100644 --- a/app/api/server/v1/emoji-custom.js +++ b/app/api/server/v1/emoji-custom.js @@ -4,6 +4,7 @@ import Busboy from 'busboy'; import { EmojiCustom } from '../../../models'; import { API } from '../api'; import { findEmojisCustom } from '../lib/emoji-custom'; +import { Media } from '../../../../server/sdk'; // DEPRECATED // Will be removed after v3.0.0 @@ -97,8 +98,14 @@ API.v1.addRoute('emoji-custom.create', { authRequired: true }, { fields.newFile = true; fields.aliases = fields.aliases || ''; try { + const emojiBuffer = Buffer.concat(emojiData); + const isUploadable = Promise.await(Media.isImage(emojiBuffer)); + if (!isUploadable) { + throw new Meteor.Error('emoji-is-not-image', 'Emoji file provided cannot be uploaded since it\'s not an image'); + } + Meteor.call('insertOrUpdateEmoji', fields); - Meteor.call('uploadEmojiCustom', Buffer.concat(emojiData), emojiMimetype, fields); + Meteor.call('uploadEmojiCustom', emojiBuffer, emojiMimetype, fields); callback(); } catch (error) { return callback(error); @@ -147,9 +154,15 @@ API.v1.addRoute('emoji-custom.update', { authRequired: true }, { fields.previousExtension = emojiToUpdate.extension; fields.aliases = fields.aliases || ''; fields.newFile = Boolean(emojiData.length); + const emojiBuffer = Buffer.concat(emojiData); + const isUploadable = Promise.await(Media.isImage(emojiBuffer)); + if (!isUploadable) { + throw new Meteor.Error('emoji-is-not-image', 'Emoji file provided cannot be uploaded since it\'s not an image'); + } + Meteor.call('insertOrUpdateEmoji', fields); if (emojiData.length) { - Meteor.call('uploadEmojiCustom', Buffer.concat(emojiData), emojiMimetype, fields); + Meteor.call('uploadEmojiCustom', emojiBuffer, emojiMimetype, fields); } callback(); } catch (error) { diff --git a/app/api/server/v1/rooms.js b/app/api/server/v1/rooms.js index 8aad5645784e..0f7d7f6ad4a8 100644 --- a/app/api/server/v1/rooms.js +++ b/app/api/server/v1/rooms.js @@ -203,6 +203,7 @@ API.v1.addRoute('rooms.cleanHistory', { authRequired: true }, { excludePinned: [true, 'true', 1, '1'].includes(this.bodyParams.excludePinned), filesOnly: [true, 'true', 1, '1'].includes(this.bodyParams.filesOnly), ignoreThreads: [true, 'true', 1, '1'].includes(this.bodyParams.ignoreThreads), + ignoreDiscussion: [true, 'true', 1, '1'].includes(this.bodyParams.ignoreDiscussion), fromUsers: this.bodyParams.users, })); diff --git a/app/api/server/v1/teams.ts b/app/api/server/v1/teams.ts index f04029234bc3..fb972a4d219a 100644 --- a/app/api/server/v1/teams.ts +++ b/app/api/server/v1/teams.ts @@ -1,5 +1,6 @@ import { Meteor } from 'meteor/meteor'; import { Promise } from 'meteor/promise'; +import { Match, check } from 'meteor/check'; import { API } from '../api'; import { Team } from '../../../../server/sdk'; @@ -77,7 +78,7 @@ API.v1.addRoute('teams.addRooms', { authRequired: true }, { } if (!hasPermission(this.userId, 'add-team-channel', team.roomId)) { - return API.v1.unauthorized(); + return API.v1.unauthorized('error-no-permission-team-channel'); } const validRooms = Promise.await(Team.addRooms(this.userId, rooms, team._id)); @@ -379,3 +380,30 @@ API.v1.addRoute('teams.autocomplete', { authRequired: true }, { return API.v1.success({ teams }); }, }); + +API.v1.addRoute('teams.update', { authRequired: true }, { + post() { + check(this.bodyParams, { + teamId: String, + data: { + name: Match.Maybe(String), + type: Match.Maybe(Number), + }, + }); + + const { teamId, data } = this.bodyParams; + + const team = teamId && Promise.await(Team.getOneById(teamId)); + if (!team) { + return API.v1.failure('team-does-not-exist'); + } + + if (!hasPermission(this.userId, 'edit-team', team.roomId)) { + return API.v1.unauthorized(); + } + + Promise.await(Team.update(this.userId, teamId, { name: data.name, type: data.type })); + + return API.v1.success(); + }, +}); diff --git a/app/api/server/v1/users.js b/app/api/server/v1/users.js index f6f5ab461509..7eb1a9006e0f 100644 --- a/app/api/server/v1/users.js +++ b/app/api/server/v1/users.js @@ -270,7 +270,7 @@ API.v1.addRoute('users.list', { authRequired: true }, { .toArray(), ); - const { sortedResults: users, totalCount: [{ total }] } = result[0]; + const { sortedResults: users, totalCount: [{ total } = { total: 0 }] } = result[0]; return API.v1.success({ users, diff --git a/app/apps/README.md b/app/apps/README.md index e314d30598d2..97047384474e 100644 --- a/app/apps/README.md +++ b/app/apps/README.md @@ -5,7 +5,7 @@ Finally! :smile: An orchestrator is the file/class which is responsible for orchestrating (starting up) everything which is required of the system to get up and going. There are two of these. One for the server and one for the client. ## What is a "Bridge"? -A bridge is a file/class which is responsible for bridging the Rocket.Chat system's data and the App system's data. They are implementations of the interfaces inside of the Rocket.Chat Apps-engine project `src/server/bridges`. They allow the two systems to talk to each other (hince the name bridge, as they "bridge the gap"). +A bridge is a file/class which is responsible for bridging the Rocket.Chat system's data and the App system's data. They are implementations of the interfaces inside of the Rocket.Chat Apps-engine project `src/server/bridges`. They allow the two systems to talk to each other (hence the name bridge, as they "bridge the gap"). ## What is a "Converter"? A converter does what the name implies, it handles converting from one system's data type into the other's. **Note**: This causes a schema to be forced on the rooms and messages. diff --git a/app/apps/server/bridges/livechat.js b/app/apps/server/bridges/livechat.js index e031479ec9da..3f4e15ad7196 100644 --- a/app/apps/server/bridges/livechat.js +++ b/app/apps/server/bridges/livechat.js @@ -218,6 +218,15 @@ export class AppLivechatBridge { return this.orch.getConverters().get('visitors').convertVisitor(LivechatVisitors.findOneVisitorByPhone(phoneNumber)); } + async findDepartmentsEnabledWithAgents(appId) { + this.orch.debugLog(`The App ${ appId } is looking for livechat departments.`); + + const converter = this.orch.getConverters().get('departments'); + const boundConverter = converter.convertDepartment.bind(converter); + + return LivechatDepartment.findEnabledWithAgents().map(boundConverter); + } + async findDepartmentByIdOrName(value, appId) { this.orch.debugLog(`The App ${ appId } is looking for livechat departments.`); diff --git a/app/apps/server/bridges/scheduler.js b/app/apps/server/bridges/scheduler.js index 89af05edbaa3..49728758f5e1 100644 --- a/app/apps/server/bridges/scheduler.js +++ b/app/apps/server/bridges/scheduler.js @@ -6,6 +6,10 @@ function _callProcessor(processor) { return (job) => processor(job?.attrs?.data || {}); } +/** + * Provides the Apps Engine with task scheduling capabilities + * It uses {@link agenda:github.com/agenda/agenda} as backend + */ export class AppSchedulerBridge { constructor(orch) { this.orch = orch; @@ -18,6 +22,34 @@ export class AppSchedulerBridge { this.isConnected = false; } + /** + * Entity that will be run in a job + * @typedef {Object} Processor + * @property {string} id The processor's identifier + * @property {function} processor The function that will be run on a given schedule + * @property {IOnetimeStartup|IRecurrentStartup} [startupSetting] If provided, the processor will be configured with the setting as soon as it gets registered + + * Processor setting for running once after being registered + * @typedef {Object} IOnetimeStartup + * @property {string} type=onetime + * @property {string} when When the processor will be executed + * @property {Object} [data] An optional object that is passed to the processor + * + * Processor setting for running recurringly after being registered + * @typedef {Object} IRecurrentStartup + * @property {string} type=recurring + * @property {string} interval When the processor will be re executed + * @property {Object} [data] An optional object that is passed to the processor + */ + + /** + * Register processors that can be scheduled to run + * + * @param {Array.} processors An array of processors + * @param {string} appId + * + * @returns Promise + */ async registerProcessors(processors = [], appId) { const runAfterRegister = []; this.orch.debugLog(`The App ${ appId } is registering job processors`, processors); @@ -30,9 +62,10 @@ export class AppSchedulerBridge { runAfterRegister.push(this.scheduleOnceAfterRegister({ id, when: startupSetting.when, data: startupSetting.data }, appId)); break; case StartupType.RECURRING: - runAfterRegister.push(this.scheduleRecurring({ id, interval: startupSetting.interval, data: startupSetting.data }, appId)); + runAfterRegister.push(this.scheduleRecurring({ id, interval: startupSetting.interval, skipImmediate: startupSetting.skipImmediate, data: startupSetting.data }, appId)); break; default: + this.orch.getRocketChatLogger().error(`Invalid startup setting type (${ startupSetting.type }) for the processor ${ id }`); break; } } @@ -43,10 +76,25 @@ export class AppSchedulerBridge { } } + /** + * Schedules a registered processor to run _once_. + * + * @param {Object} job + * @param {string} job.id The processor's id + * @param {string} job.when When the processor will be executed + * @param {Object} [job.data] An optional object that is passed to the processor + * @param {string} appId + * + * @returns Promise + */ async scheduleOnce(job, appId) { this.orch.debugLog(`The App ${ appId } is scheduling an onetime job`, job); - await this.startScheduler(); - await this.scheduler.schedule(job.when, job.id, job.data || {}); + try { + await this.startScheduler(); + await this.scheduler.schedule(job.when, job.id, job.data || {}); + } catch (e) { + this.orch.getRocketChatLogger().error(e); + } } async scheduleOnceAfterRegister(job, appId) { @@ -56,22 +104,55 @@ export class AppSchedulerBridge { } } - async scheduleRecurring(job, appId) { - this.orch.debugLog(`The App ${ appId } is scheduling a recurring job`, job); - await this.startScheduler(); - await this.scheduler.every(job.interval, job.id, job.data || {}); + /** + * Schedules a registered processor to run recurrently according to a given interval + * + * @param {Object} job + * @param {string} job.id The processor's id + * @param {string} job.interval When the processor will be re executed + * @param {boolean} job.skipImmediate=false Whether to let the first iteration to execute as soon as the task is registered + * @param {Object} [job.data] An optional object that is passed to the processor + * @param {string} appId + * + * @returns Promise + */ + async scheduleRecurring({ id, interval, skipImmediate = false, data }, appId) { + this.orch.debugLog(`The App ${ appId } is scheduling a recurring job`, id); + try { + await this.startScheduler(); + const job = this.scheduler.create(id, data || {}); + job.repeatEvery(interval, { skipImmediate }); + await job.save(); + } catch (e) { + this.orch.getRocketChatLogger().error(e); + } } + /** + * Cancels a running job given its jobId + * + * @param {string} jobId + * @param {string} appId + * + * @returns Promise + */ async cancelJob(jobId, appId) { this.orch.debugLog(`The App ${ appId } is canceling a job`, jobId); await this.startScheduler(); try { await this.scheduler.cancel({ name: jobId }); } catch (e) { - console.error(e); + this.orch.getRocketChatLogger().error(e); } } + /** + * Cancels all the running jobs from the app + * + * @param {string} appId + * + * @returns Promise + */ async cancelAllJobs(appId) { this.orch.debugLog(`Canceling all jobs of App ${ appId }`); await this.startScheduler(); @@ -79,7 +160,7 @@ export class AppSchedulerBridge { try { await this.scheduler.cancel({ name: { $regex: matcher } }); } catch (e) { - console.error(e); + this.orch.getRocketChatLogger().error(e); } } diff --git a/app/apps/server/communication/rest.js b/app/apps/server/communication/rest.js index 827fab65c5b9..06c5520b58cb 100644 --- a/app/apps/server/communication/rest.js +++ b/app/apps/server/communication/rest.js @@ -260,7 +260,9 @@ export class AppsRestApi { return API.v1.failure({ error: 'Failed to get a file to install for the App. ' }); } - const aff = Promise.await(manager.add(buff, { marketplaceInfo, permissionsGranted, enable: true })); + const user = orchestrator.getConverters().get('users').convertToApp(Meteor.user()); + + const aff = Promise.await(manager.add(buff, { marketplaceInfo, permissionsGranted, enable: true, user })); const info = aff.getAppInfo(); if (aff.hasStorageError()) { @@ -505,7 +507,9 @@ export class AppsRestApi { return API.v1.notFound(`No App found by the id of: ${ this.urlParams.id }`); } - Promise.await(manager.remove(prl.getID())); + const user = orchestrator.getConverters().get('users').convertToApp(Meteor.user()); + + Promise.await(manager.remove(prl.getID(), { user })); const info = prl.getInfo(); info.status = prl.getStatus(); diff --git a/app/apps/server/converters/departments.js b/app/apps/server/converters/departments.js index 34fc08415ffc..3d5a76a0e1e1 100644 --- a/app/apps/server/converters/departments.js +++ b/app/apps/server/converters/departments.js @@ -25,6 +25,13 @@ export class AppDepartmentsConverter { enabled: 'enabled', numberOfAgents: 'numAgents', showOnOfflineForm: 'showOnOfflineForm', + description: 'description', + offlineMessageChannelName: 'offlineMessageChannelName', + requestTagBeforeClosingChat: 'requestTagBeforeClosingChat', + chatClosingTags: 'chatClosingTags', + abandonedRoomsCloseCustomMessage: 'abandonedRoomsCloseCustomMessage', + waitingQueueMessage: 'waitingQueueMessage', + departmentsAllowedToForward: 'departmentsAllowedToForward', showOnRegistration: 'showOnRegistration', }; @@ -45,6 +52,13 @@ export class AppDepartmentsConverter { numAgents: department.numberOfAgents, showOnOfflineForm: department.showOnOfflineForm, showOnRegistration: department.showOnRegistration, + description: department.description, + offlineMessageChannelName: department.offlineMessageChannelName, + requestTagBeforeClosingChat: department.requestTagBeforeClosingChat, + chatClosingTags: department.chatClosingTags, + abandonedRoomsCloseCustomMessage: department.abandonedRoomsCloseCustomMessage, + waitingQueueMessage: department.waitingQueueMessage, + departmentsAllowedToForward: department.departmentsAllowedToForward, }; return Object.assign(newDepartment, department._unmappedProperties_); diff --git a/app/apps/server/tests/mocks/orchestrator.mock.js b/app/apps/server/tests/mocks/orchestrator.mock.js index 6429b7a182ef..55912d5a5faa 100644 --- a/app/apps/server/tests/mocks/orchestrator.mock.js +++ b/app/apps/server/tests/mocks/orchestrator.mock.js @@ -1,3 +1,5 @@ +/* istanbul ignore file */ + export class AppServerOrchestratorMock { constructor() { this._marketplaceUrl = 'https://marketplace.rocket.chat'; diff --git a/app/authorization/client/index.js b/app/authorization/client/index.js index bf484b064551..04692ed0749f 100644 --- a/app/authorization/client/index.js +++ b/app/authorization/client/index.js @@ -1,7 +1,6 @@ import { hasAllPermission, hasAtLeastOnePermission, hasPermission, userHasAllPermission } from './hasPermission'; import { hasRole } from './hasRole'; import { AuthorizationUtils } from '../lib/AuthorizationUtils'; -import './usersNameChanged'; import './requiresPermission.html'; import './startup'; diff --git a/app/authorization/client/usersNameChanged.js b/app/authorization/client/usersNameChanged.js deleted file mode 100644 index c498ca472b3c..000000000000 --- a/app/authorization/client/usersNameChanged.js +++ /dev/null @@ -1,18 +0,0 @@ -import { Meteor } from 'meteor/meteor'; - -import { Notifications } from '../../notifications'; -import { RoomRoles } from '../../models'; - -Meteor.startup(function() { - Notifications.onLogged('Users:NameChanged', function({ _id, name }) { - RoomRoles.update({ - 'u._id': _id, - }, { - $set: { - 'u.name': name, - }, - }, { - multi: true, - }); - }); -}); diff --git a/app/channel-settings/client/index.js b/app/channel-settings/client/index.js index 64b7f76be003..3e8041d42cef 100644 --- a/app/channel-settings/client/index.js +++ b/app/channel-settings/client/index.js @@ -1,5 +1,3 @@ -import './startup/messageTypes'; -import './startup/tabBar'; -import './views/Multiselect'; +import './tabBar'; export { ChannelSettings } from './lib/ChannelSettings'; diff --git a/app/channel-settings/client/startup/messageTypes.js b/app/channel-settings/client/startup/messageTypes.js deleted file mode 100644 index e693779180f2..000000000000 --- a/app/channel-settings/client/startup/messageTypes.js +++ /dev/null @@ -1,67 +0,0 @@ -import { Meteor } from 'meteor/meteor'; - -import { escapeHTML } from '../../../../lib/escapeHTML'; -import { MessageTypes } from '../../../ui-utils'; -import { t } from '../../../utils'; - -Meteor.startup(function() { - MessageTypes.registerType({ - id: 'room_changed_privacy', - system: true, - message: 'room_changed_privacy', - data(message) { - return { - user_by: message.u && message.u.username, - room_type: t(message.msg), - }; - }, - }); - - MessageTypes.registerType({ - id: 'room_changed_topic', - system: true, - message: 'room_changed_topic', - data(message) { - return { - user_by: message.u && message.u.username, - room_topic: escapeHTML(message.msg || `(${ t('None').toLowerCase() })`), - }; - }, - }); - - MessageTypes.registerType({ - id: 'room_changed_avatar', - system: true, - message: 'room_changed_avatar', - data(message) { - return { - user_by: message.u && message.u.username, - }; - }, - }); - - - MessageTypes.registerType({ - id: 'room_changed_announcement', - system: true, - message: 'room_changed_announcement', - data(message) { - return { - user_by: message.u && message.u.username, - room_announcement: escapeHTML(message.msg || `(${ t('None').toLowerCase() })`), - }; - }, - }); - - MessageTypes.registerType({ - id: 'room_changed_description', - system: true, - message: 'room_changed_description', - data(message) { - return { - user_by: message.u && message.u.username, - room_description: escapeHTML(message.msg || `(${ t('None').toLowerCase() })`), - }; - }, - }); -}); diff --git a/app/channel-settings/client/startup/tabBar.ts b/app/channel-settings/client/tabBar.ts similarity index 53% rename from app/channel-settings/client/startup/tabBar.ts rename to app/channel-settings/client/tabBar.ts index e655852ae58d..91dc8aa1fe74 100644 --- a/app/channel-settings/client/startup/tabBar.ts +++ b/app/channel-settings/client/tabBar.ts @@ -1,6 +1,6 @@ import { FC, lazy, LazyExoticComponent } from 'react'; -import { addAction } from '../../../../client/views/room/lib/Toolbox'; +import { addAction } from '../../../client/views/room/lib/Toolbox'; addAction('channel-settings', { groups: ['channel', 'group'], @@ -9,6 +9,6 @@ addAction('channel-settings', { full: true, title: 'Room_Info', icon: 'info-circled', - template: lazy(() => import('../../../../client/views/room/contextualBar/Info')) as LazyExoticComponent, - order: 7, + template: lazy(() => import('../../../client/views/room/contextualBar/Info')) as LazyExoticComponent, + order: 1, }); diff --git a/app/channel-settings/client/views/Multiselect.js b/app/channel-settings/client/views/Multiselect.js deleted file mode 100644 index 1f90cc45a38f..000000000000 --- a/app/channel-settings/client/views/Multiselect.js +++ /dev/null @@ -1,12 +0,0 @@ -import { HTML } from 'meteor/htmljs'; - -import { createTemplateForComponent } from '../../../../client/reactAdapters'; - -createTemplateForComponent( - 'Multiselect', - () => import('../../../../client/admin/settings/inputs/MultiSelectSettingInput'), - { - // eslint-disable-next-line new-cap - renderContainerView: () => HTML.DIV({ class: 'rc-multiselect', style: 'display: flex;' }), - }, -); diff --git a/app/channel-settings/server/functions/saveRoomName.js b/app/channel-settings/server/functions/saveRoomName.js index 954715a9143a..5d3197d133b3 100644 --- a/app/channel-settings/server/functions/saveRoomName.js +++ b/app/channel-settings/server/functions/saveRoomName.js @@ -1,8 +1,8 @@ import { Meteor } from 'meteor/meteor'; -import { Rooms, Messages, Subscriptions, Integrations } from '../../../models'; -import { roomTypes, getValidRoomName } from '../../../utils'; -import { callbacks } from '../../../callbacks'; +import { Rooms, Messages, Subscriptions, Integrations } from '../../../models/server'; +import { roomTypes, getValidRoomName } from '../../../utils/server'; +import { callbacks } from '../../../callbacks/server'; import { checkUsernameAvailability } from '../../../lib/server/functions'; const updateRoomName = (rid, displayName, isDiscussion) => { @@ -34,6 +34,7 @@ export const saveRoomName = function(rid, displayName, user, sendMessage = true) if (!update) { return; } + Integrations.updateRoomName(room.name, displayName); if (sendMessage) { Messages.createRoomRenamedWithRoomIdRoomNameAndUser(rid, displayName, user); diff --git a/app/channel-settings/server/functions/saveRoomType.js b/app/channel-settings/server/functions/saveRoomType.js index 58d94aa4682a..e7dd8ebb733e 100644 --- a/app/channel-settings/server/functions/saveRoomType.js +++ b/app/channel-settings/server/functions/saveRoomType.js @@ -2,9 +2,9 @@ import { Meteor } from 'meteor/meteor'; import { Match } from 'meteor/check'; import { TAPi18n } from 'meteor/rocketchat:tap-i18n'; -import { Rooms, Subscriptions, Messages } from '../../../models'; -import { settings } from '../../../settings'; -import { roomTypes, RoomSettingsEnum } from '../../../utils'; +import { Rooms, Subscriptions, Messages } from '../../../models/server'; +import { settings } from '../../../settings/server'; +import { roomTypes, RoomSettingsEnum } from '../../../utils/server'; export const saveRoomType = function(rid, roomType, user, sendMessage = true) { if (!Match.test(rid, String)) { @@ -33,7 +33,11 @@ export const saveRoomType = function(rid, roomType, user, sendMessage = true) { } const result = Rooms.setTypeById(rid, roomType) && Subscriptions.updateTypeByRoomId(rid, roomType); - if (result && sendMessage) { + if (!result) { + return result; + } + + if (sendMessage) { let message; if (roomType === 'c') { message = TAPi18n.__('Channel', { diff --git a/app/channel-settings/server/methods/saveRoomSettings.js b/app/channel-settings/server/methods/saveRoomSettings.js index e524637e8b96..811c492fb70d 100644 --- a/app/channel-settings/server/methods/saveRoomSettings.js +++ b/app/channel-settings/server/methods/saveRoomSettings.js @@ -18,6 +18,8 @@ import { saveRoomTokenpass } from '../functions/saveRoomTokens'; import { saveRoomEncrypted } from '../functions/saveRoomEncrypted'; import { saveStreamingOptions } from '../functions/saveStreamingOptions'; import { RoomSettingsEnum, roomTypes } from '../../../utils'; +import { Team } from '../../../../server/sdk'; +import { TEAM_TYPE } from '../../../../definition/ITeam'; const fields = ['roomAvatar', 'featured', 'roomName', 'roomTopic', 'roomAnnouncement', 'roomCustomFields', 'roomDescription', 'roomType', 'readOnly', 'reactWhenReadOnly', 'systemMessages', 'default', 'joinCode', 'tokenpass', 'streamingOptions', 'retentionEnabled', 'retentionMaxAge', 'retentionExcludePinned', 'retentionFilesOnly', 'retentionIgnoreThreads', 'retentionOverrideGlobal', 'encrypted', 'favorite']; @@ -125,8 +127,14 @@ const validators = { }; const settingSavers = { - roomName({ value, rid, user }) { - saveRoomName(rid, value, user); + roomName({ value, rid, user, room }) { + if (!saveRoomName(rid, value, user)) { + return; + } + + if (room.teamId && room.teamMain) { + Team.update(user._id, room.teamId, { name: value, updateRoom: false }); + } }, roomTopic({ value, room, rid, user }) { if (value !== room.topic) { @@ -149,8 +157,17 @@ const settingSavers = { } }, roomType({ value, room, rid, user }) { - if (value !== room.t) { - saveRoomType(rid, value, user); + if (value === room.t) { + return; + } + + if (!saveRoomType(rid, value, user)) { + return; + } + + if (room.teamId && room.teamMain) { + const type = value === 'c' ? TEAM_TYPE.PUBLIC : TEAM_TYPE.PRIVATE; + Team.update(user._id, room.teamId, { type, updateRoom: false }); } }, tokenpass({ value, rid }) { diff --git a/app/chatpal-search/client/route.js b/app/chatpal-search/client/route.js index 91f718f14985..158fb28458ca 100644 --- a/app/chatpal-search/client/route.js +++ b/app/chatpal-search/client/route.js @@ -1,14 +1,9 @@ -import { BlazeLayout } from 'meteor/kadira:blaze-layout'; - +import { appLayout } from '../../../client/lib/appLayout'; import { registerAdminRoute } from '../../../client/views/admin'; -import { t } from '../../utils'; registerAdminRoute('/chatpal', { name: 'chatpal-admin', action() { - return BlazeLayout.render('main', { - center: 'ChatpalAdmin', - pageTitle: t('Chatpal_AdminPage'), - }); + return appLayout.render('main', { center: 'ChatpalAdmin' }); }, }); diff --git a/app/chatpal-search/client/template/admin.html b/app/chatpal-search/client/template/admin.html index f9b1e6c83a99..91c31cf49d21 100644 --- a/app/chatpal-search/client/template/admin.html +++ b/app/chatpal-search/client/template/admin.html @@ -3,7 +3,7 @@
- {{_ pageTitle}} + {{_ "Chatpal_AdminPage"}}
diff --git a/app/discussion/client/discussionFromMessageBox.js b/app/discussion/client/discussionFromMessageBox.js index 964d1f384f07..7f210869dbbf 100644 --- a/app/discussion/client/discussionFromMessageBox.js +++ b/app/discussion/client/discussionFromMessageBox.js @@ -4,6 +4,7 @@ import { Tracker } from 'meteor/tracker'; import { messageBox, modal } from '../../ui-utils/client'; import { t } from '../../utils/client'; import { settings } from '../../settings/client'; +import { hasPermission } from '../../authorization/client'; Meteor.startup(function() { Tracker.autorun(() => { @@ -13,7 +14,7 @@ Meteor.startup(function() { messageBox.actions.add('Create_new', 'Discussion', { id: 'start-discussion', icon: 'discussion', - condition: () => true, + condition: () => hasPermission('start-discussion') || hasPermission('start-discussion-other-user'), action(data) { modal.open({ title: t('Discussion_title'), diff --git a/app/discussion/client/tabBar.ts b/app/discussion/client/tabBar.ts index 14bfe145c438..658fe5d797b9 100644 --- a/app/discussion/client/tabBar.ts +++ b/app/discussion/client/tabBar.ts @@ -15,6 +15,6 @@ addAction('discussions', () => { icon: 'discussion', template, full: true, - order: 7, + order: 3, } : null), [discussionEnabled]); }); diff --git a/app/e2e/client/tabbar.ts b/app/e2e/client/tabbar.ts index e3d23015dd3e..9ea671c9a40d 100644 --- a/app/e2e/client/tabbar.ts +++ b/app/e2e/client/tabbar.ts @@ -1,17 +1,20 @@ -import { useMemo } from 'react'; +import { useMemo, useCallback } from 'react'; import { useMutableCallback } from '@rocket.chat/fuselage-hooks'; import { addAction } from '../../../client/views/room/lib/Toolbox'; import { useSetting } from '../../../client/contexts/SettingsContext'; import { usePermission } from '../../../client/contexts/AuthorizationContext'; import { useMethod } from '../../../client/contexts/ServerContext'; +import { useReactiveValue } from '../../../client/hooks/useReactiveValue'; import { e2e } from './rocketchat.e2e'; addAction('e2e', ({ room }) => { const e2eEnabled = useSetting('E2E_Enable'); - const e2eReady = e2e.isReady() || room.encrypted; - const e2ePermission = room.t === 'd' || usePermission('toggle-room-e2e-encryption', room._id); - const hasPermission = usePermission('edit-room', room._id) && e2ePermission && e2eReady; + const e2eReady = useReactiveValue(useCallback(() => e2e.isReady(), [])) || room.encrypted; + const canToggleE2e = usePermission('toggle-room-e2e-encryption', room._id); + const canEditRoom = usePermission('edit-room', room._id); + const hasPermission = (room.t === 'd' || (canEditRoom && canToggleE2e)) && e2eReady; + const toggleE2E = useMethod('saveRoomSettings'); const action = useMutableCallback(() => { diff --git a/app/emoji-custom/server/methods/uploadEmojiCustom.js b/app/emoji-custom/server/methods/uploadEmojiCustom.js index 685d4f0270ae..fe793c9925f1 100644 --- a/app/emoji-custom/server/methods/uploadEmojiCustom.js +++ b/app/emoji-custom/server/methods/uploadEmojiCustom.js @@ -6,6 +6,7 @@ import { hasPermission } from '../../../authorization'; import { RocketChatFile } from '../../../file'; import { RocketChatFileEmojiCustomInstance } from '../startup/emoji-custom'; import { api } from '../../../../server/sdk/api'; +import { Media } from '../../../../server/sdk'; const getFile = async (file, extension) => { if (extension !== 'svg+xml') { @@ -19,6 +20,8 @@ const getFile = async (file, extension) => { Meteor.methods({ async uploadEmojiCustom(binaryContent, contentType, emojiData) { + // technically, since this method doesnt have any datatype validations, users can + // upload videos as emojis. The FE won't play them, but they will waste space for sure. if (!hasPermission(this.userId, 'manage-emoji')) { throw new Meteor.Error('not_authorized'); } @@ -28,10 +31,19 @@ Meteor.methods({ delete emojiData.aliases; const file = await getFile(Buffer.from(binaryContent, 'binary'), emojiData.extension); - emojiData.extension = emojiData.extension === 'svg+xml' ? 'png' : emojiData.extension; - const rs = RocketChatFile.bufferToStream(file); + let fileBuffer; + // sharp doesn't support these formats without imagemagick or libvips installed + // so they will be stored as they are :( + if (['gif', 'x-icon', 'bmp', 'webm'].includes(emojiData.extension)) { + fileBuffer = file; + } else { + const { data: resizedEmojiBuffer } = await Media.resizeFromBuffer(file, 128, 128, true, false, false, 'inside'); + fileBuffer = resizedEmojiBuffer; + } + + const rs = RocketChatFile.bufferToStream(fileBuffer); RocketChatFileEmojiCustomInstance.deleteFile(encodeURIComponent(`${ emojiData.name }.${ emojiData.extension }`)); const ws = RocketChatFileEmojiCustomInstance.createWriteStream(encodeURIComponent(`${ emojiData.name }.${ emojiData.extension }`), contentType); ws.on('end', Meteor.bindEnvironment(() => diff --git a/app/file-upload/server/config/FileSystem.js b/app/file-upload/server/config/FileSystem.js index 9e92499bd3dd..c0119eaca6f0 100644 --- a/app/file-upload/server/config/FileSystem.js +++ b/app/file-upload/server/config/FileSystem.js @@ -6,6 +6,9 @@ import _ from 'underscore'; import { settings } from '../../../settings'; import { FileUploadClass, FileUpload } from '../lib/FileUpload'; +import { getFileRange, setRangeHeaders } from '../lib/ranges'; + +const statSync = Meteor.wrapAsync(fs.stat); const FileSystemUploads = new FileUploadClass({ name: 'FileSystem:Uploads', @@ -14,18 +17,40 @@ const FileSystemUploads = new FileUploadClass({ get(file, req, res) { const filePath = this.store.getFilePath(file._id, file); + const options = {}; + try { - const stat = Meteor.wrapAsync(fs.stat)(filePath); + const stat = statSync(filePath); + if (!stat?.isFile()) { + res.writeHead(404); + res.end(); + return; + } - if (stat && stat.isFile()) { - file = FileUpload.addExtensionTo(file); - res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`); - res.setHeader('Last-Modified', file.uploadedAt.toUTCString()); - res.setHeader('Content-Type', file.type || 'application/octet-stream'); - res.setHeader('Content-Length', file.size); + file = FileUpload.addExtensionTo(file); + res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`); + res.setHeader('Last-Modified', file.uploadedAt.toUTCString()); + res.setHeader('Content-Type', file.type || 'application/octet-stream'); + + if (req.headers.range) { + const range = getFileRange(file, req); + + if (range) { + setRangeHeaders(range, file, res); + if (range.outOfRange) { + return; + } + options.start = range.start; + options.end = range.stop; + } + } - this.store.getReadStream(file._id, file).pipe(res); + // set content-length if range has not set + if (!res.getHeader('Content-Length')) { + res.setHeader('Content-Length', file.size); } + + this.store.getReadStream(file._id, file, options).pipe(res); } catch (e) { res.writeHead(404); res.end(); @@ -35,7 +60,7 @@ const FileSystemUploads = new FileUploadClass({ copy(file, out) { const filePath = this.store.getFilePath(file._id, file); try { - const stat = Meteor.wrapAsync(fs.stat)(filePath); + const stat = statSync(filePath); if (stat && stat.isFile()) { file = FileUpload.addExtensionTo(file); @@ -56,7 +81,7 @@ const FileSystemAvatars = new FileUploadClass({ const filePath = this.store.getFilePath(file._id, file); try { - const stat = Meteor.wrapAsync(fs.stat)(filePath); + const stat = statSync(filePath); if (stat && stat.isFile()) { file = FileUpload.addExtensionTo(file); @@ -77,7 +102,7 @@ const FileSystemUserDataFiles = new FileUploadClass({ const filePath = this.store.getFilePath(file._id, file); try { - const stat = Meteor.wrapAsync(fs.stat)(filePath); + const stat = statSync(filePath); if (stat && stat.isFile()) { file = FileUpload.addExtensionTo(file); diff --git a/app/file-upload/server/config/GridFS.js b/app/file-upload/server/config/GridFS.js index 8f04b32b20f2..a6f52d589edd 100644 --- a/app/file-upload/server/config/GridFS.js +++ b/app/file-upload/server/config/GridFS.js @@ -6,6 +6,7 @@ import { UploadFS } from 'meteor/jalik:ufs'; import { Logger } from '../../../logger'; import { FileUploadClass, FileUpload } from '../lib/FileUpload'; +import { getFileRange, setRangeHeaders } from '../lib/ranges'; const logger = new Logger('FileUpload'); @@ -50,20 +51,6 @@ ExtractRange.prototype._transform = function(chunk, enc, cb) { cb(); }; - -const getByteRange = function(header) { - if (header) { - const matches = header.match(/(\d+)-(\d+)/); - if (matches) { - return { - start: parseInt(matches[1], 10), - stop: parseInt(matches[2], 10), - }; - } - } - return null; -}; - // code from: https://github.com/jalik/jalik-ufs/blob/master/ufs-server.js#L310 const readFromGridFS = function(storeName, fileId, file, req, res) { const store = UploadFS.getStore(storeName); @@ -80,48 +67,44 @@ const readFromGridFS = function(storeName, fileId, file, req, res) { ws.emit('end'); }); - const accept = req.headers['accept-encoding'] || ''; - // Transform stream store.transformRead(rs, ws, fileId, file, req); - const range = getByteRange(req.headers.range); - let out_of_range = false; + + const range = getFileRange(file, req); if (range) { - out_of_range = (range.start > file.size) || (range.stop <= range.start) || (range.stop > file.size); + setRangeHeaders(range, file, res); + + if (range.outOfRange) { + return; + } + + logger.debug('File upload extracting range'); + ws.pipe(new ExtractRange({ start: range.start, stop: range.stop })).pipe(res); + return; } + const accept = req.headers['accept-encoding'] || ''; + // Compress data using gzip - if (accept.match(/\bgzip\b/) && range === null) { + if (accept.match(/\bgzip\b/)) { res.setHeader('Content-Encoding', 'gzip'); res.removeHeader('Content-Length'); res.writeHead(200); ws.pipe(zlib.createGzip()).pipe(res); - } else if (accept.match(/\bdeflate\b/) && range === null) { - // Compress data using deflate + return; + } + + // Compress data using deflate + if (accept.match(/\bdeflate\b/)) { res.setHeader('Content-Encoding', 'deflate'); res.removeHeader('Content-Length'); res.writeHead(200); ws.pipe(zlib.createDeflate()).pipe(res); - } else if (range && out_of_range) { - // out of range request, return 416 - res.removeHeader('Content-Length'); - res.removeHeader('Content-Type'); - res.removeHeader('Content-Disposition'); - res.removeHeader('Last-Modified'); - res.setHeader('Content-Range', `bytes */${ file.size }`); - res.writeHead(416); - res.end(); - } else if (range) { - res.setHeader('Content-Range', `bytes ${ range.start }-${ range.stop }/${ file.size }`); - res.removeHeader('Content-Length'); - res.setHeader('Content-Length', range.stop - range.start + 1); - res.writeHead(206); - logger.debug('File upload extracting range'); - ws.pipe(new ExtractRange({ start: range.start, stop: range.stop })).pipe(res); - } else { - res.writeHead(200); - ws.pipe(res); + return; } + + res.writeHead(200); + ws.pipe(res); }; const copyFromGridFS = function(storeName, fileId, file, out) { diff --git a/app/file-upload/server/lib/ranges.js b/app/file-upload/server/lib/ranges.js new file mode 100644 index 000000000000..832a262c2ffc --- /dev/null +++ b/app/file-upload/server/lib/ranges.js @@ -0,0 +1,49 @@ +export function getByteRange(header) { + if (!header) { + return; + } + const matches = header.match(/(\d+)-(\d+)/); + if (!matches) { + return; + } + return { + start: parseInt(matches[1], 10), + stop: parseInt(matches[2], 10), + }; +} + +export function getFileRange(file, req) { + const range = getByteRange(req.headers.range); + if (!range) { + return; + } + if (range.start > file.size || range.stop <= range.start || range.stop > file.size) { + return { outOfRange: true }; + } + + return { start: range.start, stop: range.stop }; +} + +// code from: https://github.com/jalik/jalik-ufs/blob/master/ufs-server.js#L310 +export const setRangeHeaders = function(range, file, res) { + if (!range) { + return; + } + + if (range.outOfRange) { + // out of range request, return 416 + res.removeHeader('Content-Length'); + res.removeHeader('Content-Type'); + res.removeHeader('Content-Disposition'); + res.removeHeader('Last-Modified'); + res.setHeader('Content-Range', `bytes */${ file.size }`); + res.writeHead(416); + res.end(); + return; + } + + res.setHeader('Content-Range', `bytes ${ range.start }-${ range.stop }/${ file.size }`); + res.removeHeader('Content-Length'); + res.setHeader('Content-Length', range.stop - range.start + 1); + res.statusCode = 206; +}; diff --git a/app/importer-csv/server/adder.js b/app/importer-csv/server/adder.js deleted file mode 100644 index da1eaacdd035..000000000000 --- a/app/importer-csv/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { CsvImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { CsvImporterInfo } from '../lib/info'; - -Importers.add(new CsvImporterInfo(), CsvImporter); diff --git a/app/importer-csv/server/importer.js b/app/importer-csv/server/importer.js index 4e113e5c5726..66cc086c1c1a 100644 --- a/app/importer-csv/server/importer.js +++ b/app/importer-csv/server/importer.js @@ -1,19 +1,11 @@ -import { Meteor } from 'meteor/meteor'; import { Random } from 'meteor/random'; -import { Accounts } from 'meteor/accounts-base'; import { - RawImports, Base, ProgressStep, - Selection, - SelectionChannel, - SelectionUser, ImporterWebsocket, } from '../../importer/server'; -import { Users, Rooms } from '../../models'; -import { insertMessage } from '../../lib'; -import { t } from '../../utils'; +import { Users } from '../../models/server'; export class CsvImporter extends Base { constructor(info, importRecord) { @@ -24,20 +16,17 @@ export class CsvImporter extends Base { prepareUsingLocalFile(fullFilePath) { this.logger.debug('start preparing import operation'); - this.collection.remove({}); + this.converter.clearImportData(); const zip = new this.AdmZip(fullFilePath); const totalEntries = zip.getEntryCount(); ImporterWebsocket.progressUpdated({ rate: 0 }); - let tempChannels = []; - let tempUsers = []; - let hasDirectMessages = false; let count = 0; let oldRate = 0; - const increaseCount = () => { + const increaseProgressCount = () => { try { count++; const rate = Math.floor(count * 1000 / totalEntries) / 10; @@ -51,44 +40,93 @@ export class CsvImporter extends Base { }; let messagesCount = 0; + let usersCount = 0; + let channelsCount = 0; + const dmRooms = new Map(); + const roomIds = new Map(); + const usedUsernames = new Set(); + const availableUsernames = new Set(); + + const getRoomId = (roomName) => { + if (!roomIds.has(roomName)) { + roomIds.set(roomName, Random.id()); + } + + return roomIds.get(roomName); + }; + zip.forEach((entry) => { this.logger.debug(`Entry: ${ entry.entryName }`); // Ignore anything that has `__MACOSX` in it's name, as sadly these things seem to mess everything up if (entry.entryName.indexOf('__MACOSX') > -1) { this.logger.debug(`Ignoring the file: ${ entry.entryName }`); - return increaseCount(); + return increaseProgressCount(); } // Directories are ignored, since they are "virtual" in a zip file if (entry.isDirectory) { this.logger.debug(`Ignoring the directory entry: ${ entry.entryName }`); - return increaseCount(); + return increaseProgressCount(); } // Parse the channels if (entry.entryName.toLowerCase() === 'channels.csv') { super.updateProgress(ProgressStep.PREPARING_CHANNELS); const parsedChannels = this.csvParser(entry.getData().toString()); - tempChannels = parsedChannels.map((c) => ({ - id: Random.id(), - name: c[0].trim(), - creator: c[1].trim(), - isPrivate: c[2].trim().toLowerCase() === 'private', - members: c[3].trim().split(';').map((m) => m.trim()), - })); - return increaseCount(); + channelsCount = parsedChannels.length; + + for (const c of parsedChannels) { + const name = c[0].trim(); + const id = getRoomId(name); + const creator = c[1].trim(); + const isPrivate = c[2].trim().toLowerCase() === 'private'; + const members = c[3].trim().split(';').map((m) => m.trim()).filter((m) => m); + + this.converter.addChannel({ + importIds: [ + id, + ], + u: { + _id: creator, + }, + name, + users: members, + t: isPrivate ? 'p' : 'c', + }); + } + + super.updateRecord({ 'count.channels': channelsCount }); + return increaseProgressCount(); } // Parse the users if (entry.entryName.toLowerCase() === 'users.csv') { super.updateProgress(ProgressStep.PREPARING_USERS); const parsedUsers = this.csvParser(entry.getData().toString()); - tempUsers = parsedUsers.map((u) => ({ id: Random.id(), username: u[0].trim(), email: u[1].trim(), name: u[2].trim() })); - - super.updateRecord({ 'count.users': tempUsers.length }); + usersCount = parsedUsers.length; + + for (const u of parsedUsers) { + const username = u[0].trim(); + availableUsernames.add(username); + + const email = u[1].trim(); + const name = u[2].trim(); + + this.converter.addUser({ + importIds: [ + username, + ], + emails: [ + email, + ], + username, + name, + }); + } - return increaseCount(); + super.updateRecord({ 'count.users': parsedUsers.length }); + return increaseProgressCount(); } // Parse the messages @@ -106,14 +144,15 @@ export class CsvImporter extends Base { msgs = this.csvParser(entry.getData().toString()); } catch (e) { this.logger.warn(`The file ${ entry.entryName } contains invalid syntax`, e); - return increaseCount(); + return increaseProgressCount(); } let data; const msgGroupData = item[1].split('.')[0]; // messages + let isDirect = false; if (folderName.toLowerCase() === 'directmessages') { - hasDirectMessages = true; + isDirect = true; data = msgs.map((m) => ({ username: m[0], ts: m[2], text: m[3], otherUsername: m[1], isDirect: true })); } else { data = msgs.map((m) => ({ username: m[0], ts: m[1], text: m[2] })); @@ -124,370 +163,83 @@ export class CsvImporter extends Base { super.updateRecord({ messagesstatus: channelName }); - if (Base.getBSONSize(data) > Base.getMaxBSONSize()) { - Base.getBSONSafeArraysFromAnArray(data).forEach((splitMsg, i) => { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: `${ channelName }.${ i }`, messages: splitMsg, channel: folderName, i, msgGroupData }); - }); - } else { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: channelName, messages: data, channel: folderName, msgGroupData }); - } - - super.updateRecord({ 'count.messages': messagesCount, messagesstatus: null }); - return increaseCount(); - } - - increaseCount(); - }); - - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'users', users: tempUsers }); - super.addCountToTotal(messagesCount + tempUsers.length); - ImporterWebsocket.progressUpdated({ rate: 100 }); - - if (hasDirectMessages) { - tempChannels.push({ - id: '#directmessages#', - name: t('Direct_Messages'), - creator: 'rocket.cat', - isPrivate: false, - isDirect: true, - members: [], - }); - } - - // Insert the channels records. - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'channels', channels: tempChannels }); - super.updateRecord({ 'count.channels': tempChannels.length }); - super.addCountToTotal(tempChannels.length); - - // Ensure we have at least a single user, channel, or message - if (tempUsers.length === 0 && tempChannels.length === 0 && messagesCount === 0) { - this.logger.error('No users, channels, or messages found in the import file.'); - super.updateProgress(ProgressStep.ERROR); - return super.getProgress(); - } - - const selectionUsers = tempUsers.map((u) => new SelectionUser(u.id, u.username, u.email, false, false, true)); - const selectionChannels = tempChannels.map((c) => new SelectionChannel(c.id, c.name, false, true, c.isPrivate, undefined, c.isDirect)); - const selectionMessages = this.importRecord.count.messages; - - super.updateProgress(ProgressStep.USER_SELECTION); - return new Selection(this.name, selectionUsers, selectionChannels, selectionMessages); - } - - startImport(importSelection) { - this.users = RawImports.findOne({ import: this.importRecord._id, type: 'users' }); - this.channels = RawImports.findOne({ import: this.importRecord._id, type: 'channels' }); - this.reloadCount(); - - const rawCollection = this.collection.model.rawCollection(); - const distinct = Meteor.wrapAsync(rawCollection.distinct, rawCollection); - - super.startImport(importSelection); - const started = Date.now(); - - // Ensure we're only going to import the users that the user has selected - for (const user of importSelection.users) { - for (const u of this.users.users) { - if (u.id === user.user_id) { - u.do_import = user.do_import; - } - } - } - this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); - - // Ensure we're only importing the channels the user has selected. - for (const channel of importSelection.channels) { - for (const c of this.channels.channels) { - if (c.id === channel.channel_id) { - c.do_import = channel.do_import; - } - } - } - this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); - - const startedByUserId = Meteor.userId(); - Meteor.defer(() => { - super.updateProgress(ProgressStep.IMPORTING_USERS); - - try { - // Import the users - for (const u of this.users.users) { - if (!u.do_import) { - continue; - } - - Meteor.runAsUser(startedByUserId, () => { - let existantUser = Users.findOneByEmailAddress(u.email); - - // If we couldn't find one by their email address, try to find an existing user by their username - if (!existantUser) { - existantUser = Users.findOneByUsernameIgnoringCase(u.username); - } - - if (existantUser) { - // since we have an existing user, let's try a few things - u.rocketId = existantUser._id; - Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } }); - } else { - const userId = Accounts.createUser({ email: u.email, password: Date.now() + u.name + u.email.toUpperCase() }); - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', u.username, { joinDefaultChannelsSilenced: true }); - Users.setName(userId, u.name); - Users.update({ _id: userId }, { $addToSet: { importIds: u.id } }); - u.rocketId = userId; - }); - } - - super.addCountCompleted(1); - }); - } - this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); - - // Import the channels - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - for (const c of this.channels.channels) { - if (!c.do_import) { - continue; - } - - if (c.isDirect) { - super.addCountCompleted(1); - continue; - } - - Meteor.runAsUser(startedByUserId, () => { - const existantRoom = Rooms.findOneByName(c.name); - // If the room exists or the name of it is 'general', then we don't need to create it again - if (existantRoom || c.name.toUpperCase() === 'GENERAL') { - c.rocketId = c.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id; - Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); - } else { - // Find the rocketchatId of the user who created this channel - let creatorId = startedByUserId; - for (const u of this.users.users) { - if (u.username === c.creator && u.do_import) { - creatorId = u.rocketId; - } - } - - // Create the channel - Meteor.runAsUser(creatorId, () => { - const roomInfo = Meteor.call(c.isPrivate ? 'createPrivateGroup' : 'createChannel', c.name, c.members); - c.rocketId = roomInfo.rid; + if (isDirect) { + for (const msg of data) { + const sourceId = [msg.username, msg.otherUsername].sort().join('/'); + + if (!dmRooms.has(sourceId)) { + this.converter.addChannel({ + importIds: [ + sourceId, + ], + users: [msg.username, msg.otherUsername], + t: 'd', }); - Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); + dmRooms.set(sourceId, true); } - super.addCountCompleted(1); - }); - } - this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); - - // If no channels file, collect channel map from DB for message-only import - if (this.channels.channels.length === 0) { - const channelNames = distinct('channel', { import: this.importRecord._id, type: 'messages', channel: { $ne: 'directMessages' } }); - for (const cname of channelNames) { - Meteor.runAsUser(startedByUserId, () => { - const existantRoom = Rooms.findOneByName(cname); - if (existantRoom || cname.toUpperCase() === 'GENERAL') { - this.channels.channels.push({ - id: cname.replace('.', '_'), - name: cname, - rocketId: cname.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id, - do_import: true, - }); - } - }); + const newMessage = { + rid: sourceId, + u: { + _id: msg.username, + }, + ts: new Date(parseInt(msg.ts)), + msg: msg.text, + }; + + usedUsernames.add(msg.username); + usedUsernames.add(msg.otherUsername); + this.converter.addMessage(newMessage); } - } - - // If no users file, collect user map from DB for message-only import - if (this.users.users.length === 0) { - const usernames = distinct('messages.username', { import: this.importRecord._id, type: 'messages' }); - for (const username of usernames) { - Meteor.runAsUser(startedByUserId, () => { - if (!this.getUserFromUsername(username)) { - const user = Users.findOneByUsernameIgnoringCase(username); - if (user) { - this.users.users.push({ - rocketId: user._id, - username: user.username, - }); - } - } - }); + } else { + const rid = getRoomId(folderName); + + for (const msg of data) { + const newMessage = { + rid, + u: { + _id: msg.username, + }, + ts: new Date(parseInt(msg.ts)), + msg: msg.text, + }; + + usedUsernames.add(msg.username); + this.converter.addMessage(newMessage); } } - // Import the Messages - super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - - const messagePacks = this.collection.find({ import: this.importRecord._id, type: 'messages' }); - messagePacks.forEach((pack) => { - const ch = pack.channel; - const { msgGroupData } = pack; - - const csvChannel = this.getChannelFromName(ch); - if (!csvChannel || !csvChannel.do_import) { - return; - } - - if (csvChannel.isDirect) { - this._importDirectMessagesFile(msgGroupData, pack, startedByUserId); - return; - } - - if (ch.toLowerCase() === 'directmessages') { - return; - } - - const room = Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); - const timestamps = {}; - - Meteor.runAsUser(startedByUserId, () => { - super.updateRecord({ messagesstatus: `${ ch }/${ msgGroupData }.${ pack.messages.length }` }); - for (const msg of pack.messages) { - if (isNaN(new Date(parseInt(msg.ts)))) { - this.logger.warn(`Timestamp on a message in ${ ch }/${ msgGroupData } is invalid`); - super.addCountCompleted(1); - continue; - } - - const creator = this.getUserFromUsername(msg.username); - if (creator) { - let suffix = ''; - if (timestamps[msg.ts] === undefined) { - timestamps[msg.ts] = 1; - } else { - suffix = `-${ timestamps[msg.ts] }`; - timestamps[msg.ts] += 1; - } - const msgObj = { - _id: `csv-${ csvChannel.id }-${ msg.ts }${ suffix }`, - ts: new Date(parseInt(msg.ts)), - msg: msg.text, - rid: room._id, - u: { - _id: creator._id, - username: creator.username, - }, - }; - - insertMessage(creator, msgObj, room, true); - } - - super.addCountCompleted(1); - } - }); - }); - - super.updateProgress(ProgressStep.FINISHING); - super.updateProgress(ProgressStep.DONE); - } catch (e) { - this.logger.error(e); - super.updateProgress(ProgressStep.ERROR); + super.updateRecord({ 'count.messages': messagesCount, messagesstatus: null }); + return increaseProgressCount(); } - const timeTook = Date.now() - started; - this.logger.log(`CSV Import took ${ timeTook } milliseconds.`); + increaseProgressCount(); }); - return super.getProgress(); - } - - _importDirectMessagesFile(msgGroupData, msgs, startedByUserId) { - const dmUsers = {}; - - const findUser = (username) => { - if (!dmUsers[username]) { - const user = this.getUserFromUsername(username) || Users.findOneByUsername(username, { fields: { username: 1 } }); - dmUsers[username] = user; - } - - return dmUsers[username]; - }; - - Meteor.runAsUser(startedByUserId, () => { - const timestamps = {}; - let room; - let rid; - super.updateRecord({ messagesstatus: `${ t('Direct_Messagest') }/${ msgGroupData }.${ msgs.messages.length }` }); - for (const msg of msgs.messages) { - if (isNaN(new Date(parseInt(msg.ts)))) { - this.logger.warn(`Timestamp on a message in ${ t('Direct_Messagest') }/${ msgGroupData } is invalid`); - super.addCountCompleted(1); - continue; - } - - const creator = findUser(msg.username); - const targetUser = findUser(msg.otherUsername); - - if (creator && targetUser) { - if (!rid) { - const roomInfo = Meteor.runAsUser(creator._id, () => Meteor.call('createDirectMessage', targetUser.username)); - rid = roomInfo.rid; - room = Rooms.findOneById(rid, { fields: { usernames: 1, t: 1, name: 1 } }); - } - - if (!room) { - this.logger.warn(`DM room not found for users ${ msg.username } and ${ msg.otherUsername }`); - super.addCountCompleted(1); - continue; - } - - let suffix = ''; - if (timestamps[msg.ts] === undefined) { - timestamps[msg.ts] = 1; - } else { - suffix = `-${ timestamps[msg.ts] }`; - timestamps[msg.ts] += 1; - } - - const msgObj = { - _id: `csv-${ rid }-${ msg.ts }${ suffix }`, - ts: new Date(parseInt(msg.ts)), - msg: msg.text, - rid: room._id, - u: { - _id: creator._id, - username: creator.username, - }, - }; - - insertMessage(creator, msgObj, room, true); - } - - super.addCountCompleted(1); + // Check if any of the message usernames was not in the imported list of users + for (const username of usedUsernames) { + if (availableUsernames.has(username)) { + continue; } - }); - } - - getChannelFromName(channelName) { - if (channelName.toLowerCase() === 'directmessages') { - return this.getDirectMessagesChannel(); - } - for (const ch of this.channels.channels) { - if (ch.name === channelName) { - return ch; + // Check if an user with that username already exists + const user = Users.findOneByUsername(username); + if (user && !user.importIds?.includes(username)) { + // Add the username to the local user's importIds so it can be found by the import process + // This way we can support importing new messages for existing users + Users.addImportIds(user._id, username); } } - } - getDirectMessagesChannel() { - for (const ch of this.channels.channels) { - if (ch.is_direct || ch.isDirect) { - return ch; - } - } - } + super.addCountToTotal(messagesCount + usersCount + channelsCount); + ImporterWebsocket.progressUpdated({ rate: 100 }); - getUserFromUsername(username) { - for (const u of this.users.users) { - if (u.username === username) { - return Users.findOneById(u.rocketId, { fields: { username: 1 } }); - } + // Ensure we have at least a single user, channel, or message + if (usersCount === 0 && channelsCount === 0 && messagesCount === 0) { + this.logger.error('No users, channels, or messages found in the import file.'); + super.updateProgress(ProgressStep.ERROR); + return super.getProgress(); } } } diff --git a/app/importer-csv/server/index.js b/app/importer-csv/server/index.js index 44a1b3bab84c..da1eaacdd035 100644 --- a/app/importer-csv/server/index.js +++ b/app/importer-csv/server/index.js @@ -1 +1,5 @@ -import './adder'; +import { CsvImporter } from './importer'; +import { Importers } from '../../importer/server'; +import { CsvImporterInfo } from '../lib/info'; + +Importers.add(new CsvImporterInfo(), CsvImporter); diff --git a/app/importer-hipchat-enterprise/server/adder.js b/app/importer-hipchat-enterprise/server/adder.js deleted file mode 100644 index 11f9a8e7b4b6..000000000000 --- a/app/importer-hipchat-enterprise/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { HipChatEnterpriseImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { HipChatEnterpriseImporterInfo } from '../lib/info'; - -Importers.add(new HipChatEnterpriseImporterInfo(), HipChatEnterpriseImporter); diff --git a/app/importer-hipchat-enterprise/server/importer.js b/app/importer-hipchat-enterprise/server/importer.js index 620315001270..3f59b669e632 100644 --- a/app/importer-hipchat-enterprise/server/importer.js +++ b/app/importer-hipchat-enterprise/server/importer.js @@ -2,21 +2,13 @@ import { Readable } from 'stream'; import path from 'path'; import fs from 'fs'; -import limax from 'limax'; import { Meteor } from 'meteor/meteor'; -import { Accounts } from 'meteor/accounts-base'; -import { Random } from 'meteor/random'; import TurndownService from 'turndown'; import { Base, ProgressStep, - Selection, - SelectionChannel, - SelectionUser, } from '../../importer/server'; -import { Messages, Users, Subscriptions, Rooms, Imports } from '../../models'; -import { insertMessage } from '../../lib'; const turndownService = new TurndownService({ strongDelimiter: '*', @@ -43,8 +35,6 @@ export class HipChatEnterpriseImporter extends Base { this.tarStream = require('tar-stream'); this.extract = this.tarStream.extract(); this.path = path; - - this.emailList = []; } parseData(data) { @@ -58,274 +48,173 @@ export class HipChatEnterpriseImporter extends Base { } } - async storeTempUsers(tempUsers) { - await this.collection.model.rawCollection().update({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - }, { - $set: { - import: this.importRecord._id, - importer: this.name, - type: 'users', - }, - $push: { - users: { $each: tempUsers }, - }, - }, { - upsert: true, - }); - - this.usersCount += tempUsers.length; - } - async prepareUsersFile(file) { super.updateProgress(ProgressStep.PREPARING_USERS); - let tempUsers = []; let count = 0; for (const u of file) { - const userData = { - id: u.User.id, - email: u.User.email, - name: u.User.name, + const newUser = { + emails: [], + importIds: [ + String(u.User.id), + ], username: u.User.mention_name, - avatar: u.User.avatar && u.User.avatar.replace(/\n/g, ''), - timezone: u.User.timezone, - isDeleted: u.User.is_deleted, + name: u.User.name, + avatarUrl: u.User.avatar && `data:image/png;base64,${ u.User.avatar.replace(/\n/g, '') }`, + bio: u.User.title || undefined, + deleted: u.User.is_deleted, + type: 'user', }; count++; if (u.User.email) { - if (this.emailList.indexOf(u.User.email) >= 0) { - userData.is_email_taken = true; - } else { - this.emailList.push(u.User.email); - } - } - - tempUsers.push(userData); - if (tempUsers.length >= 100) { - await this.storeTempUsers(tempUsers); // eslint-disable-line no-await-in-loop - tempUsers = []; + newUser.emails.push(u.User.email); } - } - if (tempUsers.length > 0) { - this.storeTempUsers(tempUsers); + this.converter.addUser(newUser); } super.updateRecord({ 'count.users': count }); super.addCountToTotal(count); } - async storeTempRooms(tempRooms) { - await this.collection.model.rawCollection().update({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - }, { - $set: { - import: this.importRecord._id, - importer: this.name, - type: 'channels', - }, - $push: { - channels: { $each: tempRooms }, - }, - }, { - upsert: true, - }); - - this.channelsCount += tempRooms.length; - } - async prepareRoomsFile(file) { super.updateProgress(ProgressStep.PREPARING_CHANNELS); - let tempRooms = []; let count = 0; for (const r of file) { - tempRooms.push({ - id: r.Room.id, - creator: r.Room.owner, - created: new Date(r.Room.created), + this.converter.addChannel({ + u: { + _id: r.Room.owner, + }, + importIds: [ + String(r.Room.id), + ], name: r.Room.name, - isPrivate: r.Room.privacy === 'private', - isArchived: r.Room.is_archived, + users: r.Room.members, + t: r.Room.privacy === 'private' ? 'p' : 'c', topic: r.Room.topic, - members: r.Room.members, + ts: new Date(r.Room.created), + archived: r.Room.is_archived, }); - count++; - - if (tempRooms.length >= 100) { - await this.storeTempRooms(tempRooms); // eslint-disable-line no-await-in-loop - tempRooms = []; - } - } - if (tempRooms.length > 0) { - await this.storeTempRooms(tempRooms); + count++; } super.updateRecord({ 'count.channels': count }); super.addCountToTotal(count); } - async storeTempMessages(tempMessages, roomIdentifier, index, subIndex, hipchatRoomId) { - this.logger.debug('dumping messages to database'); - const name = subIndex ? `${ roomIdentifier }/${ index }/${ subIndex }` : `${ roomIdentifier }/${ index }`; - - await this.collection.model.rawCollection().insert({ - import: this.importRecord._id, - importer: this.name, - type: 'messages', - name, - messages: tempMessages, - roomIdentifier, - hipchatRoomId, - }); - - this.messagesCount += tempMessages.length; - } - - async storeUserTempMessages(tempMessages, roomIdentifier, index) { - this.logger.debug(`dumping ${ tempMessages.length } messages from room ${ roomIdentifier } to database`); - await this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'user-messages', - name: `${ roomIdentifier }/${ index }`, - messages: tempMessages, - roomIdentifier, - }); - - this.messagesCount += tempMessages.length; - } - - async prepareUserMessagesFile(file, roomIdentifier, index) { - await this.loadExistingMessagesIfNecessary(); - let msgs = []; + async prepareUserMessagesFile(file) { this.logger.debug(`preparing room with ${ file.length } messages `); - for (const m of file) { - if (m.PrivateUserMessage) { - // If the message id is already on the list, skip it - if (this.preparedMessages[m.PrivateUserMessage.id] !== undefined) { - continue; - } - this.preparedMessages[m.PrivateUserMessage.id] = true; + let count = 0; + const dmRooms = []; - const newId = `hipchatenterprise-private-${ m.PrivateUserMessage.id }`; - const skipMessage = this._checkIfMessageExists(newId); - const skipAttachment = skipMessage && (m.PrivateUserMessage.attachment_path ? this._checkIfMessageExists(`${ newId }-attachment`) : true); + for (const m of file) { + if (!m.PrivateUserMessage) { + continue; + } - if (!skipMessage || !skipAttachment) { - msgs.push({ - type: 'user', - id: newId, - senderId: m.PrivateUserMessage.sender.id, - receiverId: m.PrivateUserMessage.receiver.id, - text: m.PrivateUserMessage.message.indexOf('/me ') === -1 ? m.PrivateUserMessage.message : `${ m.PrivateUserMessage.message.replace(/\/me /, '_') }_`, + // If the message id is already on the list, skip it + if (this.preparedMessages[m.PrivateUserMessage.id] !== undefined) { + continue; + } + this.preparedMessages[m.PrivateUserMessage.id] = true; + + const senderId = String(m.PrivateUserMessage.sender.id); + const receiverId = String(m.PrivateUserMessage.receiver.id); + const users = [senderId, receiverId].sort(); + + if (!dmRooms[receiverId]) { + dmRooms[receiverId] = this.converter.findDMForImportedUsers(senderId, receiverId); + + if (!dmRooms[receiverId]) { + const room = { + importIds: [ + users.join(''), + ], + users, + t: 'd', ts: new Date(m.PrivateUserMessage.timestamp.split(' ')[0]), - attachment: m.PrivateUserMessage.attachment, - attachment_path: m.PrivateUserMessage.attachment_path, - skip: skipMessage, - skipAttachment, - }); + }; + this.converter.addChannel(room); + dmRooms[receiverId] = room; } } - if (msgs.length >= 500) { - await this.storeUserTempMessages(msgs, roomIdentifier, index); // eslint-disable-line no-await-in-loop - msgs = []; - } - } - - if (msgs.length > 0) { - await this.storeUserTempMessages(msgs, roomIdentifier, index); + const rid = dmRooms[receiverId].importIds[0]; + const newMessage = this.convertImportedMessage(m.PrivateUserMessage, rid, 'private'); + count++; + this.converter.addMessage(newMessage); } - return msgs.length; + return count; } - _checkIfMessageExists(messageId) { - if (this._hasAnyImportedMessage === false) { - return false; - } + convertImportedMessage(importedMessage, rid, type) { + const idType = type === 'private' ? type : `${ rid }-${ type }`; + const newId = `hipchatenterprise-${ idType }-${ importedMessage.id }`; - return this._previewsMessagesIds.has(messageId); - } + const newMessage = { + _id: newId, + rid, + ts: new Date(importedMessage.timestamp.split(' ')[0]), + u: { + _id: String(importedMessage.sender.id), + }, + }; - async loadExistingMessagesIfNecessary() { - if (this._hasAnyImportedMessage === false) { - return false; + const text = importedMessage.message; + + if (importedMessage.message_format === 'html') { + newMessage.msg = turndownService.turndown(text); + } else if (text.startsWith('/me ')) { + newMessage.msg = `${ text.replace(/\/me /, '_') }_`; + } else { + newMessage.msg = text; } - if (!this._previewsMessagesIds) { - this._previewsMessagesIds = new Set(); - await Messages.model.rawCollection().find({}, { fields: { _id: 1 } }).forEach((i) => this._previewsMessagesIds.add(i._id)); + if (importedMessage.attachment?.url) { + const fileId = `${ importedMessage.id }-${ importedMessage.attachment.name || 'attachment' }`; + + newMessage._importFile = { + downloadUrl: importedMessage.attachment.url, + id: `${ fileId }`, + size: importedMessage.attachment.size || 0, + name: importedMessage.attachment.name, + external: false, + source: 'hipchat-enterprise', + original: { + ...importedMessage.attachment, + }, + }; } + + return newMessage; } - async prepareRoomMessagesFile(file, roomIdentifier, id, index) { - let roomMsgs = []; + async prepareRoomMessagesFile(file, rid) { this.logger.debug(`preparing room with ${ file.length } messages `); - let subIndex = 0; - - await this.loadExistingMessagesIfNecessary(); + let count = 0; for (const m of file) { if (m.UserMessage) { - const newId = `hipchatenterprise-${ id }-user-${ m.UserMessage.id }`; - const skipMessage = this._checkIfMessageExists(newId); - const skipAttachment = skipMessage && (m.UserMessage.attachment_path ? this._checkIfMessageExists(`${ newId }-attachment`) : true); - - if (!skipMessage || !skipAttachment) { - roomMsgs.push({ - type: 'user', - id: newId, - userId: m.UserMessage.sender.id, - text: m.UserMessage.message.indexOf('/me ') === -1 ? m.UserMessage.message : `${ m.UserMessage.message.replace(/\/me /, '_') }_`, - ts: new Date(m.UserMessage.timestamp.split(' ')[0]), - attachment: m.UserMessage.attachment, - attachment_path: m.UserMessage.attachment_path, - skip: skipMessage, - skipAttachment, - }); - } + const newMessage = this.convertImportedMessage(m.UserMessage, rid, 'user'); + this.converter.addMessage(newMessage); + count++; } else if (m.NotificationMessage) { - const text = m.NotificationMessage.message.indexOf('/me ') === -1 ? m.NotificationMessage.message : `${ m.NotificationMessage.message.replace(/\/me /, '_') }_`; - const newId = `hipchatenterprise-${ id }-notif-${ m.NotificationMessage.id }`; - const skipMessage = this._checkIfMessageExists(newId); - const skipAttachment = skipMessage && (m.NotificationMessage.attachment_path ? this._checkIfMessageExists(`${ newId }-attachment`) : true); + const newMessage = this.convertImportedMessage(m.NotificationMessage, rid, 'notif'); + newMessage.u._id = 'rocket.cat'; + newMessage.alias = m.NotificationMessage.sender; - if (!skipMessage || !skipAttachment) { - roomMsgs.push({ - type: 'user', - id: newId, - userId: 'rocket.cat', - alias: m.NotificationMessage.sender, - text: m.NotificationMessage.message_format === 'html' ? turndownService.turndown(text) : text, - ts: new Date(m.NotificationMessage.timestamp.split(' ')[0]), - attachment: m.NotificationMessage.attachment, - attachment_path: m.NotificationMessage.attachment_path, - skip: skipMessage, - skipAttachment, - }); - } + this.converter.addMessage(newMessage); + count++; } else if (m.TopicRoomMessage) { - const newId = `hipchatenterprise-${ id }-topic-${ m.TopicRoomMessage.id }`; - const skipMessage = this._checkIfMessageExists(newId); - if (!skipMessage) { - roomMsgs.push({ - type: 'topic', - id: newId, - userId: m.TopicRoomMessage.sender.id, - ts: new Date(m.TopicRoomMessage.timestamp.split(' ')[0]), - text: m.TopicRoomMessage.message, - skip: skipMessage, - }); - } + const newMessage = this.convertImportedMessage(m.TopicRoomMessage, rid, 'topic'); + newMessage.t = 'room_changed_topic'; + + this.converter.addMessage(newMessage); + count++; } else if (m.ArchiveRoomMessage) { this.logger.warn('Archived Room Notification was ignored.'); } else if (m.GuestAccessMessage) { @@ -333,38 +222,24 @@ export class HipChatEnterpriseImporter extends Base { } else { this.logger.error('HipChat Enterprise importer isn\'t configured to handle this message:', m); } - - if (roomMsgs.length >= 500) { - subIndex++; - await this.storeTempMessages(roomMsgs, roomIdentifier, index, subIndex, id); // eslint-disable-line no-await-in-loop - roomMsgs = []; - } - } - - if (roomMsgs.length > 0) { - await this.storeTempMessages(roomMsgs, roomIdentifier, index, subIndex > 0 ? subIndex + 1 : undefined, id); } - return roomMsgs.length; + return count; } async prepareMessagesFile(file, info) { super.updateProgress(ProgressStep.PREPARING_MESSAGES); - let messageGroupIndex = 0; - let userMessageGroupIndex = 0; - const [type, id] = info.dir.split('/'); // ['users', '1'] + const [type, id] = info.dir.split('/'); const roomIdentifier = `${ type }/${ id }`; super.updateRecord({ messagesstatus: roomIdentifier }); switch (type) { case 'users': - userMessageGroupIndex++; - return this.prepareUserMessagesFile(file, roomIdentifier, userMessageGroupIndex); + return this.prepareUserMessagesFile(file); case 'rooms': - messageGroupIndex++; - return this.prepareRoomMessagesFile(file, roomIdentifier, id, messageGroupIndex); + return this.prepareRoomMessagesFile(file, id); default: this.logger.error(`HipChat Enterprise importer isn't configured to handle "${ type }" files (${ info.dir }).`); return 0; @@ -388,215 +263,24 @@ export class HipChatEnterpriseImporter extends Base { case 'history.json': return this.prepareMessagesFile(file, info); case 'emoticons.json': - this.logger.error('HipChat Enterprise importer doesn\'t import emoticons.', info); + case 'metadata.json': break; default: - this.logger.error(`HipChat Enterprise importer doesn't know what to do with the file "${ fileName }" :o`, info); + this.logger.error(`HipChat Enterprise importer doesn't know what to do with the file "${ fileName }"`); break; } return 0; } - async _prepareFolderEntry(fullEntryPath, relativeEntryPath) { - const files = fs.readdirSync(fullEntryPath); - for (const fileName of files) { - try { - const fullFilePath = path.join(fullEntryPath, fileName); - const fullRelativePath = path.join(relativeEntryPath, fileName); - - this.logger.info(`new entry from import folder: ${ fileName }`); - - if (fs.statSync(fullFilePath).isDirectory()) { - await this._prepareFolderEntry(fullFilePath, fullRelativePath); // eslint-disable-line no-await-in-loop - continue; - } - - if (!fileName.endsWith('.json')) { - continue; - } - - let fileData; - - const promise = new Promise((resolve, reject) => { - fs.readFile(fullFilePath, (error, data) => { - if (error) { - this.logger.error(error); - return reject(error); - } - - fileData = data; - return resolve(); - }); - }); - - await promise.catch((error) => { // eslint-disable-line no-await-in-loop - this.logger.error(error); - fileData = null; - }); - - if (!fileData) { - this.logger.info(`Skipping the file: ${ fileName }`); - continue; - } - - this.logger.info(`Processing the file: ${ fileName }`); - const info = this.path.parse(fullRelativePath); - await this.prepareFile(info, fileData, fileName); // eslint-disable-line no-await-in-loop - - this.logger.debug('moving to next import folder entry'); - } catch (e) { - this.logger.debug('failed to prepare file'); - this.logger.error(e); - } - } - } - - prepareUsingLocalFolder(fullFolderPath) { - this.logger.debug('start preparing import operation using local folder'); - this.collection.remove({}); - this.emailList = []; - - this._hasAnyImportedMessage = Boolean(Messages.findOne({ _id: /hipchatenterprise\-.*/ })); - - this.usersCount = 0; - this.channelsCount = 0; - this.messagesCount = 0; - - // HipChat duplicates direct messages (one for each user) - // This object will keep track of messages that have already been prepared so it doesn't try to do it twice - this.preparedMessages = {}; - - const promise = new Promise(async (resolve, reject) => { - try { - await this._prepareFolderEntry(fullFolderPath, '.'); - this._finishPreparationProcess(resolve, reject); - } catch (e) { - this.logger.error(e); - reject(e); - } - }); - - return promise; - } - - async _finishPreparationProcess(resolve, reject) { - await this.fixPublicChannelMembers(); - - this.logger.info('finished parsing files, checking for errors now'); - this._previewsMessagesIds = undefined; - this.emailList = []; - this.preparedMessages = {}; - - - super.updateRecord({ 'count.messages': this.messagesCount, messagesstatus: null }); - super.addCountToTotal(this.messagesCount); - - // Check if any of the emails used are already taken - if (this.emailList.length > 0) { - const conflictingUsers = Users.find({ 'emails.address': { $in: this.emailList } }); - const conflictingUserEmails = []; - - conflictingUsers.forEach((conflictingUser) => { - if (conflictingUser.emails && conflictingUser.emails.length) { - conflictingUser.emails.forEach((email) => { - conflictingUserEmails.push(email.address); - }); - } - }); - - if (conflictingUserEmails.length > 0) { - this.flagConflictingEmails(conflictingUserEmails); - } - } - - // Ensure we have some users, channels, and messages - if (!this.usersCount && !this.channelsCount && !this.messagesCount) { - this.logger.info(`users: ${ this.usersCount }, channels: ${ this.channelsCount }, messages = ${ this.messagesCount }`); - super.updateProgress(ProgressStep.ERROR); - reject(new Meteor.Error('error-import-file-is-empty')); - return; - } - - const tempUsers = this.collection.findOne({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - }); - - const tempChannels = this.collection.findOne({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - }); - - const selectionUsers = tempUsers.users.map((u) => new SelectionUser(u.id, u.username, u.email, u.isDeleted, false, u.do_import !== false, u.is_email_taken === true)); - const selectionChannels = tempChannels.channels.map((r) => new SelectionChannel(r.id, r.name, r.isArchived, true, r.isPrivate, r.creator)); - const selectionMessages = this.messagesCount; - - super.updateProgress(ProgressStep.USER_SELECTION); - - resolve(new Selection(this.name, selectionUsers, selectionChannels, selectionMessages)); - } - - async fixPublicChannelMembers() { - await this.collection.model.rawCollection().aggregate([{ - $match: { - import: this.importRecord._id, - type: 'channels', - }, - }, { - $unwind: '$channels', - }, { - $match: { - 'channels.members.0': { $exists: false }, - }, - }, { - $group: { _id: '$channels.id' }, - }]).forEach(async (channel) => { - const userIds = (await this.collection.model.rawCollection().aggregate([{ - $match: { - $or: [ - { roomIdentifier: `rooms/${ channel._id }` }, - { roomIdentifier: `users/${ channel._id }` }, - ], - }, - }, { - $unwind: '$messages', - }, { - $match: { 'messages.userId': { $ne: 'rocket.cat' } }, - }, { - $group: { _id: '$messages.userId' }, - }]).toArray()).map((i) => i._id); - - await this.collection.model.rawCollection().update({ - 'channels.id': channel._id, - }, { - $set: { - 'channels.$.members': userIds, - }, - }); - }); - } - prepareUsingLocalFile(fullFilePath) { - if (fs.statSync(fullFilePath).isDirectory()) { - return this.prepareUsingLocalFolder(fullFilePath); - } - this.logger.debug('start preparing import operation'); - this.collection.remove({}); - this.emailList = []; - - this._hasAnyImportedMessage = Boolean(Messages.findOne({ _id: /hipchatenterprise\-.*/ })); - - this.usersCount = 0; - this.channelsCount = 0; - this.messagesCount = 0; + this.converter.clearImportData(); // HipChat duplicates direct messages (one for each user) // This object will keep track of messages that have already been prepared so it doesn't try to do it twice this.preparedMessages = {}; + let messageCount = 0; const promise = new Promise((resolve, reject) => { this.extract.on('entry', Meteor.bindEnvironment((header, stream, next) => { @@ -617,7 +301,12 @@ export class HipChatEnterpriseImporter extends Base { stream.on('end', Meteor.bindEnvironment(async () => { this.logger.info(`Processing the file: ${ header.name }`); - await this.prepareFile(info, data, header.name); + const newMessageCount = await this.prepareFile(info, data, header.name); + + messageCount += newMessageCount; + super.updateRecord({ 'count.messages': messageCount }); + super.addCountToTotal(newMessageCount); + data = undefined; this.logger.debug('next import entry'); @@ -634,7 +323,7 @@ export class HipChatEnterpriseImporter extends Base { }); this.extract.on('finish', Meteor.bindEnvironment(() => { - this._finishPreparationProcess(resolve, reject); + resolve(); })); const rs = fs.createReadStream(fullFilePath); @@ -650,664 +339,4 @@ export class HipChatEnterpriseImporter extends Base { return promise; } - - _saveUserIdReference(hipchatId, rocketId) { - this._userIdReference[hipchatId] = rocketId; - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - 'users.id': hipchatId, - }, { - $set: { - 'users.$.rocketId': rocketId, - }, - }); - } - - _getUserRocketId(hipchatId) { - if (!this._userIdReference) { - return; - } - - return this._userIdReference[hipchatId]; - } - - _saveRoomIdReference(hipchatId, rocketId) { - this._roomIdReference[hipchatId] = rocketId; - } - - _getRoomRocketId(hipchatId) { - if (!this._roomIdReference) { - return; - } - - return this._roomIdReference[hipchatId]; - } - - _updateImportedUser(userToImport, existingUserId) { - userToImport.rocketId = existingUserId; - this._saveUserIdReference(userToImport.id, existingUserId); - - Meteor.runAsUser(existingUserId, () => { - Users.update({ _id: existingUserId }, { - $push: { - importIds: userToImport.id, - }, - $set: { - active: userToImport.isDeleted !== true, - name: userToImport.name, - username: userToImport.username, - }, - }); - - // TODO: Think about using a custom field for the users "title" field - if (userToImport.avatar) { - Meteor.call('setAvatarFromService', `data:image/png;base64,${ userToImport.avatar }`); - } - }); - } - - _importUser(userToImport, startedByUserId) { - Meteor.runAsUser(startedByUserId, () => { - let existingUser = Users.findOneByUsernameIgnoringCase(userToImport.username); - if (!existingUser) { - // If there's no user with that username, but there's an imported user with the same original ID and no username, use that - existingUser = Users.findOne({ - importIds: userToImport.id, - username: { $exists: false }, - }); - } - - if (existingUser) { - // since we have an existing user, let's try a few things - this._saveUserIdReference(userToImport.id, existingUser._id); - userToImport.rocketId = existingUser._id; - - try { - this._updateImportedUser(userToImport, existingUser._id); - } catch (e) { - this.logger.error(e); - this.addUserError(userToImport.id, e); - } - } else { - const user = { - email: userToImport.email, - password: Random.id(), - username: userToImport.username, - name: userToImport.name, - active: userToImport.isDeleted !== true, - }; - if (!user.email) { - delete user.email; - } - if (!user.username) { - delete user.username; - } - if (!user.name) { - delete user.name; - } - - try { - const userId = Accounts.createUser(user); - - userToImport.rocketId = userId; - this._saveUserIdReference(userToImport.id, userId); - - this._updateImportedUser(userToImport, userId); - } catch (e) { - this.logger.error(e); - this.addUserError(userToImport.id, e); - } - } - - super.addCountCompleted(1); - }); - } - - _applyUserSelections(importSelection) { - // Ensure we're only going to import the users that the user has selected - const usersToImport = importSelection.users.filter((user) => user.do_import !== false).map((user) => user.user_id); - const usersNotToImport = importSelection.users.filter((user) => user.do_import === false).map((user) => user.user_id); - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - 'users.id': { - $in: usersToImport, - }, - }, { - $set: { - 'users.$.do_import': true, - }, - }); - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - 'users.id': { - $in: usersNotToImport, - }, - }, { - $set: { - 'users.$.do_import': false, - }, - }); - - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.users.user_id': { - $in: usersToImport, - }, - }, { - $set: { - 'fileData.users.$.do_import': true, - }, - }); - - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.users.user_id': { - $in: usersNotToImport, - }, - }, { - $set: { - 'fileData.users.$.do_import': false, - }, - }); - - // Ensure we're only importing the channels the user has selected. - const channelsToImport = importSelection.channels.filter((channel) => channel.do_import !== false).map((channel) => channel.channel_id); - const channelsNotToImport = importSelection.channels.filter((channel) => channel.do_import === false).map((channel) => channel.channel_id); - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - 'channels.id': { - $in: channelsToImport, - }, - }, { - $set: { - 'channels.$.do_import': true, - }, - }); - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - 'channels.id': { - $in: channelsNotToImport, - }, - }, { - $set: { - 'channels.$.do_import': false, - }, - }); - - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.channels.channel_id': { - $in: channelsToImport, - }, - }, { - $set: { - 'fileData.channels.$.do_import': true, - }, - }); - - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.channels.channel_id': { - $in: channelsNotToImport, - }, - }, { - $set: { - 'fileData.channels.$.do_import': false, - }, - }); - } - - startImport(importSelection) { - this.reloadCount(); - super.startImport(importSelection); - this._userDataCache = {}; - const started = Date.now(); - - this._applyUserSelections(importSelection); - - const startedByUserId = Meteor.userId(); - Meteor.defer(async () => { - try { - await super.updateProgress(ProgressStep.IMPORTING_USERS); - await this._importUsers(startedByUserId); - - await super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - await this._importChannels(startedByUserId); - - await super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - await this._importMessages(startedByUserId); - await this._importDirectMessages(); - - // super.updateProgress(ProgressStep.FINISHING); - await super.updateProgress(ProgressStep.DONE); - } catch (e) { - super.updateRecord({ 'error-record': JSON.stringify(e, Object.getOwnPropertyNames(e)) }); - this.logger.error(e); - super.updateProgress(ProgressStep.ERROR); - } - - const timeTook = Date.now() - started; - this.logger.log(`HipChat Enterprise Import took ${ timeTook } milliseconds.`); - this._userDataCache = {}; - this._userIdReference = {}; - this._roomIdReference = {}; - }); - - return super.getProgress(); - } - - _importUsers(startedByUserId) { - this._userIdReference = {}; - - const userLists = this.collection.find({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - }); - - userLists.forEach((list) => { - if (!list.users) { - return; - } - - list.users.forEach((u) => { - this.logger.debug(`Starting the user import: ${ u.username } and are we importing them? ${ u.do_import }`); - if (u.do_import === false) { - return; - } - - this._importUser(u, startedByUserId); - }); - }); - } - - _createSubscriptions(channelToImport, roomOrRoomId) { - if (!channelToImport || !channelToImport.members) { - return; - } - - let room; - if (roomOrRoomId && typeof roomOrRoomId === 'string') { - room = Rooms.findOneByIdOrName(roomOrRoomId); - } else { - room = roomOrRoomId; - } - - const extra = { open: true }; - channelToImport.members.forEach((hipchatUserId) => { - if (hipchatUserId === channelToImport.creator) { - // Creators are subscribed automatically - return; - } - - const user = this.getRocketUserFromUserId(hipchatUserId); - if (!user) { - this.logger.error(`User ${ hipchatUserId } not found on Rocket.Chat database.`); - return; - } - - if (Subscriptions.find({ rid: room._id, 'u._id': user._id }, { limit: 1 }).count() === 0) { - this.logger.info(`Creating user's subscription to room ${ room._id }, rocket.chat user is ${ user._id }, hipchat user is ${ hipchatUserId }`); - Subscriptions.createWithRoomAndUser(room, user, extra); - } - }); - } - - _importChannel(channelToImport, startedByUserId) { - Meteor.runAsUser(startedByUserId, () => { - const existingRoom = Rooms.findOneByName(limax(channelToImport.name)); - // If the room exists or the name of it is 'general', then we don't need to create it again - if (existingRoom || channelToImport.name.toUpperCase() === 'GENERAL') { - channelToImport.rocketId = channelToImport.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existingRoom._id; - this._saveRoomIdReference(channelToImport.id, channelToImport.rocketId); - Rooms.update({ _id: channelToImport.rocketId }, { $push: { importIds: channelToImport.id } }); - - this._createSubscriptions(channelToImport, existingRoom || 'general'); - } else { - // Find the rocketchatId of the user who created this channel - const creatorId = this._getUserRocketId(channelToImport.creator) || startedByUserId; - - // Create the channel - Meteor.runAsUser(creatorId, () => { - try { - const roomInfo = Meteor.call(channelToImport.isPrivate ? 'createPrivateGroup' : 'createChannel', channelToImport.name, []); - this._saveRoomIdReference(channelToImport.id, roomInfo.rid); - channelToImport.rocketId = roomInfo.rid; - } catch (e) { - this.logger.error(`Failed to create channel, using userId: ${ creatorId };`, e); - } - }); - - if (channelToImport.rocketId) { - Rooms.update({ _id: channelToImport.rocketId }, { $set: { ts: channelToImport.created, topic: channelToImport.topic }, $push: { importIds: channelToImport.id } }); - this._createSubscriptions(channelToImport, channelToImport.rocketId); - } - } - - super.addCountCompleted(1); - }); - } - - _importChannels(startedByUserId) { - this._roomIdReference = {}; - const channelLists = this.collection.find({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - }); - - channelLists.forEach((list) => { - if (!list.channels) { - return; - } - list.channels.forEach((c) => { - this.logger.debug(`Starting the channel import: ${ c.name } and are we importing them? ${ c.do_import }`); - if (c.do_import === false) { - return; - } - - this._importChannel(c, startedByUserId); - }); - }); - } - - _importAttachment(msg, room, sender) { - if (msg.attachment_path && !msg.skipAttachment) { - const details = { - message_id: `${ msg.id }-attachment`, - name: msg.attachment.name, - size: msg.attachment.size, - userId: sender._id, - rid: room._id, - }; - - this.uploadFile(details, msg.attachment.url, sender, room, msg.ts); - } - } - - _importSingleMessage(msg, roomIdentifier, room) { - if (isNaN(msg.ts)) { - this.logger.error(`Timestamp on a message in ${ roomIdentifier } is invalid`); - return; - } - - try { - const creator = this.getRocketUserFromUserId(msg.userId); - if (creator) { - Meteor.runAsUser(creator._id, () => { - this._importAttachment(msg, room, creator); - - switch (msg.type) { - case 'user': - if (!msg.skip) { - insertMessage(creator, { - _id: msg.id, - ts: msg.ts, - msg: msg.text, - rid: room._id, - alias: msg.alias, - u: { - _id: creator._id, - username: creator.username, - }, - }, room, false); - } - break; - case 'topic': - Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_topic', room._id, msg.text, creator, { _id: msg.id, ts: msg.ts }); - break; - } - }); - } else { - this.logger.error(`Hipchat user not found: ${ msg.userId }`); - this.addMessageError(new Meteor.Error('error-message-sender-is-invalid'), `Hipchat user not found: ${ msg.userId }`); - } - } catch (e) { - this.logger.error(e); - this.addMessageError(e, msg); - } - } - - async _importMessageList(startedByUserId, messageListId) { - const list = this.collection.findOneById(messageListId); - if (!list) { - return; - } - - if (!list.messages) { - return; - } - - const { roomIdentifier, hipchatRoomId, name } = list; - const rid = await this._getRoomRocketId(hipchatRoomId); - - // If there's no rocketId for the channel, then it wasn't imported - if (!rid) { - this.logger.debug(`Ignoring room ${ roomIdentifier } ( ${ name } ), as there's no rid to use.`); - return; - } - - const room = await Rooms.findOneById(rid, { fields: { usernames: 1, t: 1, name: 1 } }); - await super.updateRecord({ - messagesstatus: `${ roomIdentifier }.${ list.messages.length }`, - 'count.completed': this.progress.count.completed, - }); - - await Meteor.runAsUser(startedByUserId, async () => { - let msgCount = 0; - try { - for (const msg of list.messages) { - await this._importSingleMessage(msg, roomIdentifier, room); // eslint-disable-line no-await-in-loop - msgCount++; - if (msgCount >= 50) { - super.addCountCompleted(msgCount); - msgCount = 0; - } - } - } catch (e) { - this.logger.error(e); - } - - if (msgCount > 0) { - super.addCountCompleted(msgCount); - } - }); - } - - async _importMessages(startedByUserId) { - const messageListIds = this.collection.find({ - import: this.importRecord._id, - importer: this.name, - type: 'messages', - }, { fields: { _id: true } }).fetch(); - - for (const item of messageListIds) { - await this._importMessageList(startedByUserId, item._id); // eslint-disable-line no-await-in-loop - } - } - - _importDirectMessages() { - const messageListIds = this.collection.find({ - import: this.importRecord._id, - importer: this.name, - type: 'user-messages', - }, { fields: { _id: true } }).fetch(); - - this.logger.info(`${ messageListIds.length } lists of messages to import.`); - - // HipChat duplicates direct messages (one for each user) - // This object will keep track of messages that have already been imported so it doesn't try to insert them twice - const importedMessages = {}; - - messageListIds.forEach((item) => { - this.logger.debug(`New list of user messages: ${ item._id }`); - const list = this.collection.findOneById(item._id); - if (!list) { - this.logger.error('Record of user-messages list not found'); - return; - } - - if (!list.messages) { - this.logger.error('No message list found on record.'); - return; - } - - const { roomIdentifier } = list; - if (!this.getRocketUserFromRoomIdentifier(roomIdentifier)) { - this.logger.error(`Skipping ${ list.messages.length } messages due to missing room ( ${ roomIdentifier } ).`); - return; - } - - this.logger.debug(`${ list.messages.length } messages on this list`); - super.updateRecord({ - messagesstatus: `${ list.name }.${ list.messages.length }`, - 'count.completed': this.progress.count.completed, - }); - - let msgCount = 0; - const roomUsers = {}; - const roomObjects = {}; - - list.messages.forEach((msg) => { - msgCount++; - if (isNaN(msg.ts)) { - this.logger.error(`Timestamp on a message in ${ list.name } is invalid`); - return; - } - - // make sure the message sender is a valid user inside rocket.chat - if (!(msg.senderId in roomUsers)) { - roomUsers[msg.senderId] = this.getRocketUserFromUserId(msg.senderId); - } - - if (!roomUsers[msg.senderId]) { - this.logger.error(`Skipping message due to missing sender ( ${ msg.senderId } ).`); - return; - } - - // make sure the receiver of the message is a valid rocket.chat user - if (!(msg.receiverId in roomUsers)) { - roomUsers[msg.receiverId] = this.getRocketUserFromUserId(msg.receiverId); - } - - if (!roomUsers[msg.receiverId]) { - this.logger.error(`Skipping message due to missing receiver ( ${ msg.receiverId } ).`); - return; - } - - const sender = roomUsers[msg.senderId]; - const receiver = roomUsers[msg.receiverId]; - - const roomId = [receiver._id, sender._id].sort().join(''); - if (!(roomId in roomObjects)) { - roomObjects[roomId] = Rooms.findOneById(roomId); - } - - let room = roomObjects[roomId]; - if (!room) { - this.logger.debug('DM room not found, creating it.'); - Meteor.runAsUser(sender._id, () => { - const roomInfo = Meteor.call('createDirectMessage', receiver.username); - - room = Rooms.findOneById(roomInfo.rid); - roomObjects[roomId] = room; - }); - } - - try { - Meteor.runAsUser(sender._id, () => { - if (importedMessages[msg.id] !== undefined) { - return; - } - importedMessages[msg.id] = true; - - if (msg.attachment_path) { - if (!msg.skipAttachment) { - this.logger.debug('Uploading DM file'); - const details = { - message_id: `${ msg.id }-attachment`, - name: msg.attachment.name, - size: msg.attachment.size, - userId: sender._id, - rid: room._id, - }; - this.uploadFile(details, msg.attachment.url, sender, room, msg.ts); - } - } - - if (!msg.skip) { - this.logger.debug('Inserting DM message'); - insertMessage(sender, { - _id: msg.id, - ts: msg.ts, - msg: msg.text, - rid: room._id, - u: { - _id: sender._id, - username: sender.username, - }, - }, room, false); - } - }); - } catch (e) { - console.error(e); - this.addMessageError(e, msg); - } - - if (msgCount >= 50) { - super.addCountCompleted(msgCount); - msgCount = 0; - } - }); - - if (msgCount > 0) { - super.addCountCompleted(msgCount); - } - }); - } - - _getBasicUserData(userId) { - if (this._userDataCache[userId]) { - return this._userDataCache[userId]; - } - - this._userDataCache[userId] = Users.findOneById(userId, { fields: { username: 1 } }); - return this._userDataCache[userId]; - } - - getRocketUserFromUserId(userId) { - if (userId === 'rocket.cat') { - return this._getBasicUserData('rocket.cat'); - } - - const rocketId = this._getUserRocketId(userId); - if (rocketId) { - return this._getBasicUserData(rocketId); - } - } - - getRocketUserFromRoomIdentifier(roomIdentifier) { - const userParts = roomIdentifier.split('/'); - if (!userParts || !userParts.length) { - return; - } - - const userId = userParts[userParts.length - 1]; - return this.getRocketUserFromUserId(userId); - } } diff --git a/app/importer-hipchat-enterprise/server/index.js b/app/importer-hipchat-enterprise/server/index.js index 44a1b3bab84c..11f9a8e7b4b6 100644 --- a/app/importer-hipchat-enterprise/server/index.js +++ b/app/importer-hipchat-enterprise/server/index.js @@ -1 +1,5 @@ -import './adder'; +import { HipChatEnterpriseImporter } from './importer'; +import { Importers } from '../../importer/server'; +import { HipChatEnterpriseImporterInfo } from '../lib/info'; + +Importers.add(new HipChatEnterpriseImporterInfo(), HipChatEnterpriseImporter); diff --git a/app/importer-hipchat/client/adder.js b/app/importer-hipchat/client/adder.js deleted file mode 100644 index 813169cc932e..000000000000 --- a/app/importer-hipchat/client/adder.js +++ /dev/null @@ -1,4 +0,0 @@ -import { Importers } from '../../importer/client'; -import { HipChatImporterInfo } from '../lib/info'; - -Importers.add(new HipChatImporterInfo()); diff --git a/app/importer-hipchat/client/index.js b/app/importer-hipchat/client/index.js deleted file mode 100644 index 44a1b3bab84c..000000000000 --- a/app/importer-hipchat/client/index.js +++ /dev/null @@ -1 +0,0 @@ -import './adder'; diff --git a/app/importer-hipchat/lib/info.js b/app/importer-hipchat/lib/info.js deleted file mode 100644 index befd126868dc..000000000000 --- a/app/importer-hipchat/lib/info.js +++ /dev/null @@ -1,7 +0,0 @@ -import { ImporterInfo } from '../../importer/lib/ImporterInfo'; - -export class HipChatImporterInfo extends ImporterInfo { - constructor() { - super('hipchat', 'HipChat (zip)', 'application/zip'); - } -} diff --git a/app/importer-hipchat/server/adder.js b/app/importer-hipchat/server/adder.js deleted file mode 100644 index 7379752371af..000000000000 --- a/app/importer-hipchat/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { HipChatImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { HipChatImporterInfo } from '../lib/info'; - -Importers.add(new HipChatImporterInfo(), HipChatImporter); diff --git a/app/importer-hipchat/server/importer.js b/app/importer-hipchat/server/importer.js deleted file mode 100644 index acf521f2305d..000000000000 --- a/app/importer-hipchat/server/importer.js +++ /dev/null @@ -1,375 +0,0 @@ -import limax from 'limax'; -import { Meteor } from 'meteor/meteor'; -import { Accounts } from 'meteor/accounts-base'; -import _ from 'underscore'; -import moment from 'moment'; - -import { - RawImports, - Base, - ProgressStep, - Selection, - SelectionChannel, - SelectionUser, -} from '../../importer/server'; -import { RocketChatFile } from '../../file'; -import { Users, Rooms } from '../../models'; -import { sendMessage } from '../../lib'; - -import 'moment-timezone'; - -export class HipChatImporter extends Base { - constructor(info, importRecord) { - super(info, importRecord); - - this.userTags = []; - this.roomPrefix = 'hipchat_export/rooms/'; - this.usersPrefix = 'hipchat_export/users/'; - } - - prepare(dataURI, sentContentType, fileName, skipTypeCheck) { - super.prepare(dataURI, sentContentType, fileName, skipTypeCheck); - const { image } = RocketChatFile.dataURIParse(dataURI); - // const contentType = ref.contentType; - const zip = new this.AdmZip(Buffer.from(image, 'base64')); - const zipEntries = zip.getEntries(); - let tempRooms = []; - let tempUsers = []; - const tempMessages = {}; - - zipEntries.forEach((entry) => { - if (entry.entryName.indexOf('__MACOSX') > -1) { - this.logger.debug(`Ignoring the file: ${ entry.entryName }`); - } - if (entry.isDirectory) { - return; - } - if (entry.entryName.indexOf(this.roomPrefix) > -1) { - let roomName = entry.entryName.split(this.roomPrefix)[1]; - if (roomName === 'list.json') { - super.updateProgress(ProgressStep.PREPARING_CHANNELS); - tempRooms = JSON.parse(entry.getData().toString()).rooms; - tempRooms.forEach((room) => { - room.name = limax(room.name); - }); - } else if (roomName.indexOf('/') > -1) { - const item = roomName.split('/'); - roomName = limax(item[0]); - const msgGroupData = item[1].split('.')[0]; - if (!tempMessages[roomName]) { - tempMessages[roomName] = {}; - } - try { - tempMessages[roomName][msgGroupData] = JSON.parse(entry.getData().toString()); - return tempMessages[roomName][msgGroupData]; - } catch (error) { - return this.logger.warn(`${ entry.entryName } is not a valid JSON file! Unable to import it.`); - } - } - } else if (entry.entryName.indexOf(this.usersPrefix) > -1) { - const usersName = entry.entryName.split(this.usersPrefix)[1]; - if (usersName === 'list.json') { - super.updateProgress(ProgressStep.PREPARING_USERS); - tempUsers = JSON.parse(entry.getData().toString()).users; - return tempUsers; - } - return this.logger.warn(`Unexpected file in the ${ this.name } import: ${ entry.entryName }`); - } - }); - const usersId = this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - users: tempUsers, - }); - this.users = this.collection.findOne(usersId); - this.updateRecord({ - 'count.users': tempUsers.length, - }); - this.addCountToTotal(tempUsers.length); - const channelsId = this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - channels: tempRooms, - }); - this.channels = this.collection.findOne(channelsId); - this.updateRecord({ - 'count.channels': tempRooms.length, - }); - this.addCountToTotal(tempRooms.length); - super.updateProgress(ProgressStep.PREPARING_MESSAGES); - let messagesCount = 0; - - Object.keys(tempMessages).forEach((channel) => { - const messagesObj = tempMessages[channel]; - - Object.keys(messagesObj).forEach((date) => { - const msgs = messagesObj[date]; - messagesCount += msgs.length; - this.updateRecord({ - messagesstatus: `${ channel }/${ date }`, - }); - - if (Base.getBSONSize(msgs) > Base.getMaxBSONSize()) { - Base.getBSONSafeArraysFromAnArray(msgs).forEach((splitMsg, i) => { - this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'messages', - name: `${ channel }/${ date }.${ i }`, - messages: splitMsg, - channel, - date, - i, - }); - }); - } else { - this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'messages', - name: `${ channel }/${ date }`, - messages: msgs, - channel, - date, - }); - } - }); - }); - this.updateRecord({ - 'count.messages': messagesCount, - messagesstatus: null, - }); - this.addCountToTotal(messagesCount); - if (tempUsers.length === 0 || tempRooms.length === 0 || messagesCount === 0) { - this.logger.warn(`The loaded users count ${ tempUsers.length }, the loaded channels ${ tempRooms.length }, and the loaded messages ${ messagesCount }`); - super.updateProgress(ProgressStep.ERROR); - return this.getProgress(); - } - const selectionUsers = tempUsers.map(function(user) { - return new SelectionUser(user.user_id, user.name, user.email, user.is_deleted, false, !user.is_bot); - }); - const selectionChannels = tempRooms.map(function(room) { - return new SelectionChannel(room.room_id, room.name, room.is_archived, true, false); - }); - const selectionMessages = this.importRecord.count.messages; - super.updateProgress(ProgressStep.USER_SELECTION); - return new Selection(this.name, selectionUsers, selectionChannels, selectionMessages); - } - - startImport(importSelection) { - this.users = RawImports.findOne({ import: this.importRecord._id, type: 'users' }); - this.channels = RawImports.findOne({ import: this.importRecord._id, type: 'channels' }); - this.reloadCount(); - - super.startImport(importSelection); - const start = Date.now(); - - importSelection.users.forEach((user) => { - this.users.users.forEach((u) => { - if (u.user_id === user.user_id) { - u.do_import = user.do_import; - } - }); - }); - this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); - - importSelection.channels.forEach((channel) => - this.channels.channels.forEach((c) => { - if (c.room_id === channel.channel_id) { - c.do_import = channel.do_import; - } - }), - ); - this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); - - const startedByUserId = Meteor.userId(); - Meteor.defer(() => { - super.updateProgress(ProgressStep.IMPORTING_USERS); - - try { - this.users.users.forEach((user) => { - if (!user.do_import) { - return; - } - - Meteor.runAsUser(startedByUserId, () => { - const existantUser = Users.findOneByEmailAddress(user.email); - if (existantUser) { - user.rocketId = existantUser._id; - this.userTags.push({ - hipchat: `@${ user.mention_name }`, - rocket: `@${ existantUser.username }`, - }); - } else { - const userId = Accounts.createUser({ - email: user.email, - password: Date.now() + user.name + user.email.toUpperCase(), - }); - user.rocketId = userId; - this.userTags.push({ - hipchat: `@${ user.mention_name }`, - rocket: `@${ user.mention_name }`, - }); - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', user.mention_name, { - joinDefaultChannelsSilenced: true, - }); - Meteor.call('setAvatarFromService', user.photo_url, undefined, 'url'); - return Meteor.call('userSetUtcOffset', parseInt(moment().tz(user.timezone).format('Z').toString().split(':')[0])); - }); - if (user.name != null) { - Users.setName(userId, user.name); - } - if (user.is_deleted) { - Meteor.call('setUserActiveStatus', userId, false); - } - } - return this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); - - const channelNames = []; - - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - this.channels.channels.forEach((channel) => { - if (!channel.do_import) { - return; - } - - channelNames.push(channel.name); - Meteor.runAsUser(startedByUserId, () => { - channel.name = channel.name.replace(/ /g, ''); - const existantRoom = Rooms.findOneByName(channel.name); - if (existantRoom) { - channel.rocketId = existantRoom._id; - } else { - let userId = ''; - this.users.users.forEach((user) => { - if (user.user_id === channel.owner_user_id) { - userId = user.rocketId; - } - }); - if (userId === '') { - this.logger.warn(`Failed to find the channel creator for ${ channel.name }, setting it to the current running user.`); - userId = startedByUserId; - } - Meteor.runAsUser(userId, () => { - const returned = Meteor.call('createChannel', channel.name, []); - channel.rocketId = returned.rid; - }); - Rooms.update({ - _id: channel.rocketId, - }, { - $set: { - ts: new Date(channel.created * 1000), - }, - }); - } - return this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); - - super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - const nousers = {}; - - for (const channel of channelNames) { - const hipchatChannel = this.getHipChatChannelFromName(channel); - - if (!hipchatChannel || !hipchatChannel.do_import) { - continue; - } - - const room = Rooms.findOneById(hipchatChannel.rocketId, { - fields: { - usernames: 1, - t: 1, - name: 1, - }, - }); - - const messagePacks = this.collection.find({ import: this.importRecord._id, type: 'messages', channel }); - - Meteor.runAsUser(startedByUserId, () => { - messagePacks.forEach((pack) => { - const packId = pack.i ? `${ pack.date }.${ pack.i }` : pack.date; - - this.updateRecord({ messagesstatus: `${ channel }/${ packId } (${ pack.messages.length })` }); - pack.messages.forEach((message) => { - if (message.from != null) { - const user = this.getRocketUser(message.from.user_id); - if (user != null) { - const msgObj = { - msg: this.convertHipChatMessageToRocketChat(message.message), - ts: new Date(message.date), - u: { - _id: user._id, - username: user.username, - }, - }; - sendMessage(user, msgObj, room, true); - } else if (!nousers[message.from.user_id]) { - nousers[message.from.user_id] = message.from; - } - } else if (!_.isArray(message)) { - console.warn('Please report the following:', message); - } - - this.addCountCompleted(1); - }); - }); - }); - } - - this.logger.warn('The following did not have users:', nousers); - super.updateProgress(ProgressStep.FINISHING); - - this.channels.channels.forEach((channel) => { - if (channel.do_import && channel.is_archived) { - Meteor.runAsUser(startedByUserId, () => Meteor.call('archiveRoom', channel.rocketId)); - } - }); - - super.updateProgress(ProgressStep.DONE); - } catch (e) { - this.logger.error(e); - super.updateProgress(ProgressStep.ERROR); - } - - const timeTook = Date.now() - start; - return this.logger.log(`Import took ${ timeTook } milliseconds.`); - }); - - return this.getProgress(); - } - - getHipChatChannelFromName(channelName) { - return this.channels.channels.find((channel) => channel.name === channelName); - } - - getRocketUser(hipchatId) { - const user = this.users.users.find((user) => user.user_id === hipchatId); - return user ? Users.findOneById(user.rocketId, { - fields: { - username: 1, - name: 1, - }, - }) : undefined; - } - - convertHipChatMessageToRocketChat(message) { - if (message != null) { - this.userTags.forEach((userReplace) => { - message = message.replace(userReplace.hipchat, userReplace.rocket); - }); - } else { - message = ''; - } - return message; - } -} diff --git a/app/importer-hipchat/server/index.js b/app/importer-hipchat/server/index.js deleted file mode 100644 index 44a1b3bab84c..000000000000 --- a/app/importer-hipchat/server/index.js +++ /dev/null @@ -1 +0,0 @@ -import './adder'; diff --git a/app/importer-pending-avatars/server/importer.js b/app/importer-pending-avatars/server/importer.js index 5d034a8c7d37..7a8767ba582a 100644 --- a/app/importer-pending-avatars/server/importer.js +++ b/app/importer-pending-avatars/server/importer.js @@ -8,12 +8,6 @@ import { import { Users } from '../../models'; export class PendingAvatarImporter extends Base { - constructor(info, importRecord) { - super(info, importRecord); - this.userTags = []; - this.bots = {}; - } - prepareFileCount() { this.logger.debug('start preparing import operation'); super.updateProgress(ProgressStep.PREPARING_STARTED); diff --git a/app/importer-slack-users/server/adder.js b/app/importer-slack-users/server/adder.js deleted file mode 100644 index 1651465e5d4e..000000000000 --- a/app/importer-slack-users/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { SlackUsersImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { SlackUsersImporterInfo } from '../lib/info'; - -Importers.add(new SlackUsersImporterInfo(), SlackUsersImporter); diff --git a/app/importer-slack-users/server/index.js b/app/importer-slack-users/server/index.js index 44a1b3bab84c..1651465e5d4e 100644 --- a/app/importer-slack-users/server/index.js +++ b/app/importer-slack-users/server/index.js @@ -1 +1,5 @@ -import './adder'; +import { SlackUsersImporter } from './importer'; +import { Importers } from '../../importer/server'; +import { SlackUsersImporterInfo } from '../lib/info'; + +Importers.add(new SlackUsersImporterInfo(), SlackUsersImporter); diff --git a/app/importer-slack/server/adder.js b/app/importer-slack/server/adder.js deleted file mode 100644 index d8499b517f00..000000000000 --- a/app/importer-slack/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { SlackImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { SlackImporterInfo } from '../lib/info'; - -Importers.add(new SlackImporterInfo(), SlackImporter); diff --git a/app/importer-slack/server/importer.js b/app/importer-slack/server/importer.js index a996985ff06f..6aeb3fd1343e 100644 --- a/app/importer-slack/server/importer.js +++ b/app/importer-slack/server/importer.js @@ -1,30 +1,17 @@ -import { Meteor } from 'meteor/meteor'; -import { Accounts } from 'meteor/accounts-base'; import _ from 'underscore'; import { - RawImports, Base, ProgressStep, - Selection, - SelectionChannel, - SelectionUser, + ImportData, ImporterWebsocket, } from '../../importer/server'; -import { getUserAvatarURL } from '../../utils/lib/getUserAvatarURL'; -import { Users, Rooms, Messages } from '../../models'; -import { insertMessage, createDirectRoom } from '../../lib'; -import { getValidRoomName } from '../../utils'; +import { Messages } from '../../models'; import { settings } from '../../settings/server'; import { MentionsParser } from '../../mentions/lib/MentionsParser'; +import { getUserAvatarURL } from '../../utils/lib/getUserAvatarURL'; export class SlackImporter extends Base { - constructor(info, importRecord) { - super(info, importRecord); - this.userTags = []; - this.bots = {}; - } - parseData(data) { const dataString = data.toString(); try { @@ -36,107 +23,238 @@ export class SlackImporter extends Base { } } + prepareChannelsFile(entry) { + super.updateProgress(ProgressStep.PREPARING_CHANNELS); + const data = JSON.parse(entry.getData().toString()).filter((channel) => channel.creator != null); + + this.logger.debug(`loaded ${ data.length } channels.`); + + this.addCountToTotal(data.length); + + for (const channel of data) { + this.converter.addChannel({ + _id: channel.is_general ? 'general' : undefined, + u: { + _id: this._replaceSlackUserId(channel.creator), + }, + importIds: [ + channel.id, + ], + name: channel.name, + users: this._replaceSlackUserIds(channel.members), + t: 'c', + topic: channel.topic?.value || undefined, + description: channel.purpose?.value || undefined, + ts: channel.created ? new Date(channel.created * 1000) : undefined, + archived: channel.is_archived, + }); + } + + return data.length; + } + + prepareGroupsFile(entry) { + super.updateProgress(ProgressStep.PREPARING_CHANNELS); + const data = JSON.parse(entry.getData().toString()).filter((channel) => channel.creator != null); + + this.logger.debug(`loaded ${ data.length } groups.`); + + this.addCountToTotal(data.length); + + for (const channel of data) { + this.converter.addChannel({ + u: { + _id: this._replaceSlackUserId(channel.creator), + }, + importIds: [ + channel.id, + ], + name: channel.name, + users: this._replaceSlackUserIds(channel.members), + t: 'p', + topic: channel.topic?.value || undefined, + description: channel.purpose?.value || undefined, + ts: channel.created ? new Date(channel.created * 1000) : undefined, + archived: channel.is_archived, + }); + } + + return data.length; + } + + prepareMpimpsFile(entry) { + super.updateProgress(ProgressStep.PREPARING_CHANNELS); + const data = JSON.parse(entry.getData().toString()).filter((channel) => channel.creator != null); + + this.logger.debug(`loaded ${ data.length } mpims.`); + + this.addCountToTotal(data.length); + + const maxUsers = settings.get('DirectMesssage_maxUsers') || 1; + + for (const channel of data) { + this.converter.addChannel({ + u: { + _id: this._replaceSlackUserId(channel.creator), + }, + importIds: [ + channel.id, + ], + name: channel.name, + users: this._replaceSlackUserIds(channel.members), + t: channel.members.length > maxUsers ? 'p' : 'd', + topic: channel.topic?.value || undefined, + description: channel.purpose?.value || undefined, + ts: channel.created ? new Date(channel.created * 1000) : undefined, + archived: channel.is_archived, + }); + } + + return data.length; + } + + prepareDMsFile(entry) { + super.updateProgress(ProgressStep.PREPARING_CHANNELS); + const data = JSON.parse(entry.getData().toString()); + + this.logger.debug(`loaded ${ data.length } dms.`); + + this.addCountToTotal(data.length); + for (const channel of data) { + this.converter.addChannel({ + importIds: [ + channel.id, + ], + users: this._replaceSlackUserIds(channel.members), + t: 'd', + ts: channel.created ? new Date(channel.created * 1000) : undefined, + }); + } + + return data.length; + } + + prepareUsersFile(entry) { + super.updateProgress(ProgressStep.PREPARING_USERS); + const data = JSON.parse(entry.getData().toString()); + + this.logger.debug(`loaded ${ data.length } users.`); + + // Insert the users record + this.updateRecord({ 'count.users': data.length }); + this.addCountToTotal(data.length); + + + for (const user of data) { + const newUser = { + emails: [], + importIds: [ + user.id, + ], + username: user.name, + name: user.profile.real_name, + utcOffset: user.tz_offset && (user.tz_offset / 3600), + avatarUrl: user.profile.image_original || user.profile.image_512, + deleted: user.deleted, + statusText: user.profile.status_text || undefined, + bio: user.profile.title || undefined, + type: 'user', + }; + + if (user.profile.email) { + newUser.emails.push(user.profile.email); + } + + if (user.is_bot) { + newUser.roles = ['bot']; + newUser.type = 'bot'; + } + + this.converter.addUser(newUser); + } + } + prepareUsingLocalFile(fullFilePath) { this.logger.debug('start preparing import operation'); - this.collection.remove({}); + this.converter.clearImportData(); const zip = new this.AdmZip(fullFilePath); const totalEntries = zip.getEntryCount(); - let tempChannels = []; - let tempGroups = []; - let tempMpims = []; - let tempDMs = []; - let tempUsers = []; let messagesCount = 0; + let channelCount = 0; let count = 0; ImporterWebsocket.progressUpdated({ rate: 0 }); let oldRate = 0; - const prepareChannelsFile = (entry, typeName, filterInvalidCreators = true) => { - super.updateProgress(ProgressStep.PREPARING_CHANNELS); - let data = JSON.parse(entry.getData().toString()); - - if (filterInvalidCreators) { - data = data.filter((channel) => channel.creator != null); - } - - this.logger.debug(`loaded ${ data.length } ${ typeName }.`); - - // Insert the channels records. - if (Base.getBSONSize(data) > Base.getMaxBSONSize()) { - const tmp = Base.getBSONSafeArraysFromAnArray(data); - Object.keys(tmp).forEach((i) => { - const splitChannels = tmp[i]; - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: typeName, name: `${ typeName }/${ i }`, channels: splitChannels, i }); - }); - } else { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: typeName, channels: data }); + const increaseProgress = () => { + try { + count++; + const rate = Math.floor(count * 1000 / totalEntries) / 10; + if (rate > oldRate) { + ImporterWebsocket.progressUpdated({ rate }); + oldRate = rate; + } + } catch (e) { + console.error(e); } - this.updateRecord({ 'count.channels': tempGroups.length + tempChannels.length + tempDMs.length + tempMpims.length + data.length }); - this.addCountToTotal(data.length); - return data; }; try { + // we need to iterate the zip file twice so that all channels are loaded before the messages + zip.forEach((entry) => { try { - if (entry.entryName.includes('__MACOSX') || entry.entryName.includes('.DS_Store')) { - return this.logger.debug(`Ignoring the file: ${ entry.entryName }`); - } - if (entry.entryName === 'channels.json') { - tempChannels = prepareChannelsFile(entry, 'channels'); - return; + channelCount += this.prepareChannelsFile(entry); + this.updateRecord({ 'count.channels': channelCount }); + return increaseProgress(); } if (entry.entryName === 'groups.json') { - tempGroups = prepareChannelsFile(entry, 'groups'); - return; + channelCount += this.prepareGroupsFile(entry); + this.updateRecord({ 'count.channels': channelCount }); + return increaseProgress(); } if (entry.entryName === 'mpims.json') { - tempMpims = prepareChannelsFile(entry, 'mpims'); - return; + channelCount += this.prepareMpimpsFile(entry); + this.updateRecord({ 'count.channels': channelCount }); + return increaseProgress(); } if (entry.entryName === 'dms.json') { - tempDMs = prepareChannelsFile(entry, 'DMs', false); - return; + channelCount += this.prepareDMsFile(entry); + this.updateRecord({ 'count.channels': channelCount }); + return increaseProgress(); } if (entry.entryName === 'users.json') { - super.updateProgress(ProgressStep.PREPARING_USERS); - tempUsers = JSON.parse(entry.getData().toString()); - - tempUsers.forEach((user) => { - if (user.is_bot) { - this.bots[user.profile.bot_id] = user; - } - }); - - this.logger.debug(`loaded ${ tempUsers.length } users.`); + this.prepareUsersFile(entry); + return increaseProgress(); + } + } catch (e) { + this.logger.error(e); + } + }); - // Insert the users record - if (Base.getBSONSize(tempUsers) > Base.getMaxBSONSize()) { - const tmp = Base.getBSONSafeArraysFromAnArray(tempUsers); - Object.keys(tmp).forEach((i) => { - const splitUsers = tmp[i]; - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'users', name: `users/${ i }`, users: splitUsers, i }); - }); - } else { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'users', name: 'users', users: tempUsers }); - } + const missedTypes = {}; + // If we have no slack message yet, then we can insert them instead of upserting + this._useUpsert = !Messages.findOne({ _id: /slack\-.*/ }); - this.updateRecord({ 'count.users': tempUsers.length }); - this.addCountToTotal(tempUsers.length); + zip.forEach((entry) => { + try { + if (entry.entryName.includes('__MACOSX') || entry.entryName.includes('.DS_Store')) { + count++; + return this.logger.debug(`Ignoring the file: ${ entry.entryName }`); + } - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'bots', bots: this.bots }); + if (['channels.json', 'groups.json', 'mpims.json', 'dms.json', 'users.json'].includes(entry.entryName)) { return; } - if (!entry.isDirectory && entry.entryName.indexOf('/') > -1) { + if (!entry.isDirectory && entry.entryName.includes('/')) { const item = entry.entryName.split('/'); const channel = item[0]; @@ -153,14 +271,12 @@ export class SlackImporter extends Base { this.updateRecord({ messagesstatus: `${ channel }/${ date }` }); this.addCountToTotal(tempMessages.length); - if (Base.getBSONSize(tempMessages) > Base.getMaxBSONSize()) { - const tmp = Base.getBSONSafeArraysFromAnArray(tempMessages); - Object.keys(tmp).forEach((i) => { - const splitMsg = tmp[i]; - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: `${ channel }/${ date }.${ i }`, messages: splitMsg, channel, date, i }); - }); - } else { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: `${ channel }/${ date }`, messages: tempMessages, channel, date }); + const slackChannelId = ImportData.findChannelImportIdByNameOrImportId(channel); + + if (slackChannelId) { + for (const message of tempMessages) { + this.prepareMessageObject(message, missedTypes, slackChannelId); + } } } catch (error) { this.logger.warn(`${ entry.entryName } is not a valid JSON file! Unable to import it.`); @@ -170,17 +286,12 @@ export class SlackImporter extends Base { this.logger.error(e); } - try { - count++; - const rate = Math.floor(count * 1000 / totalEntries) / 10; - if (rate > oldRate) { - ImporterWebsocket.progressUpdated({ rate }); - oldRate = rate; - } - } catch (e) { - console.error(e); - } + increaseProgress(); }); + + if (!_.isEmpty(missedTypes)) { + console.log('Missed import types:', missedTypes); + } } catch (e) { this.logger.error(e); throw e; @@ -188,271 +299,99 @@ export class SlackImporter extends Base { ImporterWebsocket.progressUpdated({ rate: 100 }); this.updateRecord({ 'count.messages': messagesCount, messagesstatus: null }); - const roomCount = tempChannels.length + tempGroups.length + tempDMs.length + tempMpims.length; - - if ([tempUsers.length, roomCount, messagesCount].some((e) => e === 0)) { - this.logger.warn(`Loaded ${ tempUsers.length } users, ${ tempChannels.length } channels, ${ tempGroups.length } groups, ${ tempDMs.length } DMs, ${ tempMpims.length } multi party IMs and ${ messagesCount } messages`); - super.updateProgress(ProgressStep.ERROR); - return this.getProgress(); - } - - const selectionUsers = (() => { - if (tempUsers.length <= 500) { - return tempUsers.map((user) => new SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot)); - } - - return [ - new SelectionUser('users', 'Regular Users', '', false, false, true), - new SelectionUser('bot_users', 'Bot Users', '', false, true, false), - new SelectionUser('deleted_users', 'Deleted Users', '', true, false, true), - ]; - })(); - - const selectionChannels = (() => { - if (roomCount <= 500) { - return tempChannels.map((channel) => new SelectionChannel(channel.id, channel.name, channel.is_archived, true, false)); - } - - return [ - new SelectionChannel('channels', 'Regular Channels', false, true, false), - new SelectionChannel('archived_channels', 'Archived Channels', true, true, false), - ]; - })(); - - const selectionGroups = (() => { - if (roomCount <= 500) { - return tempGroups.map((channel) => new SelectionChannel(channel.id, channel.name, channel.is_archived, true, true)); - } - - return [ - new SelectionChannel('groups', 'Regular Groups', false, true, true), - new SelectionChannel('archived_groups', 'Archived Groups', true, true, true), - ]; - })(); - - const selectionMpims = [ - new SelectionChannel('mpims', 'Multi Party DMs', false, true, true), - new SelectionChannel('archived_mimps', 'Archived Multi Party DMs', true, true, true), - ]; - - const selectionMessages = this.importRecord.count.messages; - super.updateProgress(ProgressStep.USER_SELECTION); - - return new Selection(this.name, selectionUsers, selectionMpims.concat(selectionChannels).concat(selectionGroups), selectionMessages); } - performUserImport(user, startedByUserId) { - if (user.is_bot) { - this._saveUserIdReference(user.id, 'rocket.cat', user.name, 'rocket.cat'); - } - - if (!user.do_import) { - this.addCountCompleted(1); - return; - } - - Meteor.runAsUser(startedByUserId, () => { - const existantUser = Users.findOneByEmailAddress(user.profile.email) || Users.findOneByUsernameIgnoringCase(user.name); - if (existantUser) { - user.rocketId = existantUser._id; - Users.update({ _id: user.rocketId }, { $addToSet: { importIds: user.id } }); - this._saveUserIdReference(user.id, existantUser._id, user.name, existantUser.username); - } else { - const userId = user.profile.email ? Accounts.createUser({ email: user.profile.email, password: Date.now() + user.name + user.profile.email.toUpperCase() }) : Accounts.createUser({ username: user.name, password: Date.now() + user.name, joinDefaultChannelsSilenced: true }); - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', user.name, { joinDefaultChannelsSilenced: true }); - - const url = user.profile.image_original || user.profile.image_512; - if (url) { - try { - Users.update({ _id: userId }, { $set: { _pendingAvatarUrl: url } }); - } catch (error) { - this.logger.warn(`Failed to set ${ user.name }'s avatar from url ${ url }`); - console.log(`Failed to set ${ user.name }'s avatar from url ${ url }`); - } - } - - // Slack's is -18000 which translates to Rocket.Chat's after dividing by 3600 - if (user.tz_offset) { - Meteor.call('userSetUtcOffset', user.tz_offset / 3600); - } - }); - - Users.update({ _id: userId }, { $addToSet: { importIds: user.id } }); - - if (user.profile.real_name) { - Users.setName(userId, user.profile.real_name); - } - - // Deleted users are 'inactive' users in Rocket.Chat - if (user.deleted) { - Meteor.call('setUserActiveStatus', userId, false); - } - - user.rocketId = userId; - this._saveUserIdReference(user.id, userId, user.name, user.name); - } - - this.addCountCompleted(1); - }); - } - - parseMentions(message) { + parseMentions(newMessage) { const mentionsParser = new MentionsParser({ pattern: () => settings.get('UTF8_Names_Validation'), useRealName: () => settings.get('UI_Use_Real_Name'), me: () => 'me', }); - if (!message.mentions) { - message.mentions = []; - } - const users = mentionsParser.getUserMentions(message.msg); - users.forEach((user_id, index, arr) => { - const user = user_id.slice(1, user_id.length); - try { - if (user === 'all' || user === 'here') { - arr[index] = user; - } else { - arr[index] = Users.findOneByUsernameIgnoringCase(user); - } - } catch (e) { - this.logger.warn(`Failed to import user mention with name: ${ user }`); + const users = mentionsParser.getUserMentions(newMessage.msg).filter((u) => u).map((uid) => this._replaceSlackUserId(uid.slice(1, uid.length))); + if (users.length) { + if (!newMessage.mentions) { + newMessage.mentions = []; } - }); - - const filteredUsers = users.filter((u) => u); - message.mentions.push(...filteredUsers); - - if (!message.channels) { - message.channels = []; + newMessage.mentions.push(...users); } - const channels = mentionsParser.getChannelMentions(message.msg); - channels.forEach((channel_name, index, arr) => { - const chan = channel_name.slice(1, channel_name.length); - try { - const slackChannel = this.getSlackChannelFromName(chan); - arr[index] = Rooms.findOneById(slackChannel.rocketId); - arr[index].dname = chan; // Have to store name to display so parser can match it - } catch (e) { - this.logger.warn(`Failed to import channel mention with name: ${ chan }`); - } - }); - const filteredChannels = channels.filter((c) => c); - message.channels.push(...filteredChannels); + const channels = mentionsParser.getChannelMentions(newMessage.msg).filter((c) => c).map((name) => name.slice(1, name.length)); + if (channels.length) { + if (!newMessage.channels) { + newMessage.channels = []; + } + newMessage.channels.push(...channels); + } } - processMessageSubType(message, room, msgDataDefaults, missedTypes) { + processMessageSubType(message, slackChannelId, newMessage, missedTypes) { const ignoreTypes = { bot_add: true, file_comment: true, file_mention: true }; - let rocketUser = this.getRocketUserFromUserId(message.user); - const useRocketCat = !rocketUser; - - if (useRocketCat) { - rocketUser = this.getRocketUserFromUserId('rocket.cat'); - } - - if (!rocketUser) { - return; - } - switch (message.subtype) { case 'channel_join': case 'group_join': - if (!useRocketCat) { - Messages.createUserJoinWithRoomIdAndUser(room._id, rocketUser, msgDataDefaults); - } - break; + newMessage.t = 'uj'; + newMessage.groupable = false; + return true; case 'channel_leave': case 'group_leave': - if (!useRocketCat) { - Messages.createUserLeaveWithRoomIdAndUser(room._id, rocketUser, msgDataDefaults); - } - break; - case 'me_message': { - const msgObj = { - ...msgDataDefaults, - msg: `_${ this.convertSlackMessageToRocketChat(message.text) }_`, - }; - this.parseMentions(msgObj); - insertMessage(rocketUser, msgObj, room, this._anyExistingSlackMessage); - break; - } - case 'bot_message': - case 'slackbot_response': { - const botUser = this.getRocketUserFromUserId('rocket.cat'); - const botUsername = this.bots[message.bot_id] ? this.bots[message.bot_id].name : message.username; - const msgObj = { - ...msgDataDefaults, - msg: this.convertSlackMessageToRocketChat(message.text), - rid: room._id, - bot: true, - attachments: this.convertMessageAttachments(message.attachments), - username: botUsername || undefined, - }; - - if (message.edited) { - msgObj.editedAt = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000); - const editedBy = this.getRocketUserFromUserId(message.edited.user); - if (editedBy) { - msgObj.editedBy = { - _id: editedBy._id, - username: editedBy.username, - }; - } - } - - if (message.icons) { - msgObj.emoji = message.icons.emoji; - } - this.parseMentions(msgObj); - insertMessage(botUser, msgObj, room, this._anyExistingSlackMessage); - break; - } - + newMessage.t = 'ul'; + newMessage.groupable = false; + return true; case 'channel_purpose': case 'group_purpose': - Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_description', room._id, message.purpose, rocketUser, msgDataDefaults); - break; + newMessage.t = 'room_changed_description'; + newMessage.groupable = false; + newMessage.msg = message.purpose; + return true; case 'channel_topic': case 'group_topic': - Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_topic', room._id, message.topic, rocketUser, msgDataDefaults); - break; + newMessage.t = 'room_changed_topic'; + newMessage.groupable = false; + newMessage.msg = message.topic; + return true; case 'channel_name': case 'group_name': - Messages.createRoomRenamedWithRoomIdRoomNameAndUser(room._id, message.name, rocketUser, msgDataDefaults); - break; + newMessage.t = 'r'; + newMessage.msg = message.name; + newMessage.groupable = false; + return true; case 'pinned_item': if (message.attachments) { - const msgObj = { - ...msgDataDefaults, - attachments: [{ - text: this.convertSlackMessageToRocketChat(message.attachments[0].text), - author_name: message.attachments[0].author_subname, - author_icon: getUserAvatarURL(message.attachments[0].author_subname), - }], - }; - - Messages.createWithTypeRoomIdMessageAndUser('message_pinned', room._id, '', rocketUser, msgObj); - } else { - // TODO: make this better - this.logger.debug('Pinned item with no attachment, needs work.'); - // Messages.createWithTypeRoomIdMessageAndUser 'message_pinned', room._id, '', @getRocketUserFromUserId(message.user), msgDataDefaults + if (!newMessage.attachments) { + newMessage.attachments = []; + } + newMessage.attachments.push({ + text: this.convertSlackMessageToRocketChat(message.attachments[0].text), + author_name: message.attachments[0].author_subname, + author_icon: getUserAvatarURL(message.attachments[0].author_subname), + }); + newMessage.t = 'message_pinned'; } break; case 'file_share': - if (message.file && message.file.url_private_download !== undefined) { - const details = { - message_id: `slack-${ message.ts.replace(/\./g, '-') }`, - name: message.file.name, - size: message.file.size, - type: message.file.mimetype, - rid: room._id, + if (message.file?.url_private_download) { + const fileId = this.makeSlackMessageId(slackChannelId, message.ts, 'share'); + const fileMessage = { + _id: fileId, + rid: newMessage.rid, + ts: newMessage.ts, + msg: message.file.url_private_download || '', + _importFile: this.convertSlackFileToPendingFile(message.file), + u: { + _id: newMessage.u._id, + }, }; - this.uploadFile(details, message.file.url_private_download, rocketUser, room, new Date(parseInt(message.ts.split('.')[0]) * 1000)); + + if (message.thread_ts && (message.thread_ts !== message.ts)) { + fileMessage.tmid = this.makeSlackMessageId(slackChannelId, message.thread_ts); + } + + this.converter.addMessage(fileMessage, this._useUpsert); } break; + default: if (!missedTypes[message.subtype] && !ignoreTypes[message.subtype]) { missedTypes[message.subtype] = message; @@ -461,690 +400,154 @@ export class SlackImporter extends Base { } } - performMessageImport(message, room, missedTypes, slackChannel) { - const msgDataDefaults = { - _id: `slack-${ slackChannel.id }-${ message.ts.replace(/\./g, '-') }`, + makeSlackMessageId(channelId, ts, fileIndex = undefined) { + const base = `slack-${ channelId }-${ ts.replace(/\./g, '-') }`; + + if (fileIndex) { + return `${ base }-file${ fileIndex }`; + } + + return base; + } + + prepareMessageObject(message, missedTypes, slackChannelId) { + const id = this.makeSlackMessageId(slackChannelId, message.ts); + const newMessage = { + _id: id, + rid: slackChannelId, ts: new Date(parseInt(message.ts.split('.')[0]) * 1000), + u: { + _id: this._replaceSlackUserId(message.user), + }, }; // Process the reactions if (message.reactions && message.reactions.length > 0) { - msgDataDefaults.reactions = {}; + newMessage.reactions = new Map(); message.reactions.forEach((reaction) => { - reaction.name = `:${ reaction.name }:`; - msgDataDefaults.reactions[reaction.name] = { usernames: [] }; - - if (reaction.users) { - reaction.users.forEach((u) => { - const rcUser = this.getRocketUserFromUserId(u); - if (!rcUser) { return; } - - msgDataDefaults.reactions[reaction.name].usernames.push(rcUser.username); + const name = `:${ reaction.name }:`; + if (reaction.users && reaction.users.length) { + newMessage.reactions.set(name, { + name, + users: this._replaceSlackUserIds(reaction.users), }); } - - if (msgDataDefaults.reactions[reaction.name].usernames.length === 0) { - delete msgDataDefaults.reactions[reaction.name]; - } }); } if (message.type === 'message') { if (message.files) { - const fileUser = this.getRocketUserFromUserId(message.user); let fileIndex = 0; - message.files.forEach((file) => { fileIndex++; - const msgObj = { - _id: `slack-${ slackChannel.id }-${ message.ts.replace(/\./g, '-') }-file${ fileIndex }`, - ts: msgDataDefaults.ts, + + const fileId = this.makeSlackMessageId(slackChannelId, message.ts, fileIndex); + const fileMessage = { + _id: fileId, + rid: slackChannelId, + ts: newMessage.ts, msg: file.url_private_download || '', _importFile: this.convertSlackFileToPendingFile(file), - }; - if (message.thread_ts && (message.thread_ts !== message.ts)) { - msgObj.tmid = `slack-${ slackChannel.id }-${ message.thread_ts.replace(/\./g, '-') }`; - } - try { - insertMessage(fileUser, msgObj, room, this._anyExistingSlackMessage); - } catch (e) { - this.logger.warn(`Failed to import the message file: ${ msgDataDefaults._id }-${ fileIndex }`); - this.logger.error(e); - } - }); - } - - if (message.subtype && (message.subtype !== 'thread_broadcast')) { - this.processMessageSubType(message, room, msgDataDefaults, missedTypes); - } else { - const user = this.getRocketUserFromUserId(message.user); - if (user) { - const msgObj = { - ...msgDataDefaults, - msg: this.convertSlackMessageToRocketChat(message.text), - rid: room._id, - attachments: this.convertMessageAttachments(message.attachments), u: { - _id: user._id, - username: user.username, + _id: this._replaceSlackUserId(message.user), }, }; - if (message.thread_ts) { - if (message.thread_ts === message.ts) { - if (message.reply_users) { - msgObj.replies = []; - message.reply_users.forEach(function(item) { - msgObj.replies.push(item); - }); - } else if (message.replies) { - msgObj.replies = []; - message.replies.forEach(function(item) { - msgObj.replies.push(item.user); - }); - } else { - this.logger.warn(`Failed to import the parent comment, message: ${ msgDataDefaults._id }. Missing replies/reply_users field`); - } - - msgObj.tcount = message.reply_count; - msgObj.tlm = new Date(parseInt(message.latest_reply.split('.')[0]) * 1000); - } else { - msgObj.tmid = `slack-${ slackChannel.id }-${ message.thread_ts.replace(/\./g, '-') }`; - } - } - - if (message.edited) { - msgObj.editedAt = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000); - const editedBy = this.getRocketUserFromUserId(message.edited.user); - if (editedBy) { - msgObj.editedBy = { - _id: editedBy._id, - username: editedBy.username, - }; - } - } - - this.parseMentions(msgObj); - try { - insertMessage(this.getRocketUserFromUserId(message.user), msgObj, room, this._anyExistingSlackMessage); - } catch (e) { - this.logger.warn(`Failed to import the message: ${ msgDataDefaults._id }`); - this.logger.error(e); - } - } - } - } - - this.addCountCompleted(1); - } - - _saveUserIdReference(slackId, rocketId, slackUsername, rocketUsername) { - this._userIdReference[slackId] = rocketId; - - this.userTags.push({ - slack: `<@${ slackId }>`, - slackLong: `<@${ slackId }|${ slackUsername }>`, - rocket: `@${ rocketUsername }`, - }); - } - - _getUserRocketId(slackId) { - if (!this._userIdReference) { - return; - } - - return this._userIdReference[slackId]; - } - - _importUsers(startedByUserId) { - this._userIdReference = {}; - - super.updateProgress(ProgressStep.IMPORTING_USERS); - for (const list of this.userLists) { - list.users.forEach((user) => this.performUserImport(user, startedByUserId)); - this.collection.update({ _id: list._id }, { $set: { users: list.users } }); - } - } - - _importChannels(startedByUserId, channelNames) { - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - - for (const list of this.channelsLists) { - list.channels.forEach((channel) => { - if (!channel.do_import) { - this.addCountCompleted(1); - return; - } - - if (channelNames.includes(channel.name)) { - this.logger.warn(`Duplicated channel name will be skipped: ${ channel.name }`); - return; - } - channelNames.push(channel.name); - - Meteor.runAsUser(startedByUserId, () => { - const existingRoom = this._findExistingRoom(channel.name); - - if (existingRoom || channel.is_general) { - if (channel.is_general && existingRoom && channel.name !== existingRoom.name) { - Meteor.call('saveRoomSettings', 'GENERAL', 'roomName', channel.name); - } - - channel.rocketId = channel.is_general ? 'GENERAL' : existingRoom._id; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } else { - const users = this._getChannelUserList(channel); - const userId = this.getImportedRocketUserIdFromSlackUserId(channel.creator) || startedByUserId; - - Meteor.runAsUser(userId, () => { - const returned = Meteor.call('createChannel', channel.name, users); - channel.rocketId = returned.rid; - }); - - this._updateImportedChannelTopicAndDescription(channel); + if (message.thread_ts && (message.thread_ts !== message.ts)) { + fileMessage.tmid = this.makeSlackMessageId(slackChannelId, message.thread_ts); } - this.addCountCompleted(1); + this.converter.addMessage(fileMessage, this._useUpsert); }); - }); - - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - } - - _findExistingRoom(name) { - const existingRoom = Rooms.findOneByName(name); - - // If no room with that name is found, try applying name rules and searching again - if (!existingRoom) { - const newName = getValidRoomName(name, null, { allowDuplicates: true }); - - if (newName !== name) { - return Rooms.findOneByName(newName); } - } - return existingRoom; - } - - _getChannelUserList(channel, returnObject = false, includeCreator = false) { - return channel.members.reduce((ret, member) => { - if (includeCreator || member !== channel.creator) { - const user = this.getRocketUserFromUserId(member); - // Don't add bots to the room's member list; Since they are all replaced with rocket.cat, it could cause duplicated subscriptions - if (user && user.username && user._id !== 'rocket.cat') { - if (returnObject) { - ret.push(user); - } else { - ret.push(user.username); - } - } - } - return ret; - }, []); - } + const regularTypes = [ + 'me_message', + 'thread_broadcast', + ]; - _importPrivateGroupList(startedByUserId, listList, channelNames) { - for (const list of listList) { - list.channels.forEach((channel) => { - if (!channel.do_import) { - this.addCountCompleted(1); - return; - } + const isBotMessage = message.subtype && ['bot_message', 'slackbot_response'].includes(message.subtype); - if (channelNames.includes(channel.name)) { - this.logger.warn(`Duplicated group name will be skipped: ${ channel.name }`); - return; + if (message.subtype && !regularTypes.includes(message.subtype) && !isBotMessage) { + if (this.processMessageSubType(message, slackChannelId, newMessage, missedTypes)) { + this.converter.addMessage(newMessage, this._useUpsert); } + } else { + const text = this.convertSlackMessageToRocketChat(message.text); - channelNames.push(channel.name); - - Meteor.runAsUser(startedByUserId, () => { - const existingRoom = this._findExistingRoom(channel.name); - - if (existingRoom) { - channel.rocketId = existingRoom._id; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } else { - const users = this._getChannelUserList(channel); - - const userId = this.getImportedRocketUserIdFromSlackUserId(channel.creator) || startedByUserId; - Meteor.runAsUser(userId, () => { - const returned = Meteor.call('createPrivateGroup', channel.name, users); - channel.rocketId = returned.rid; - }); - - this._updateImportedChannelTopicAndDescription(channel); - } - - this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - } - - _importGroups(startedByUserId, channelNames) { - this._importPrivateGroupList(startedByUserId, this.groupsLists, channelNames); - } - - _updateImportedChannelTopicAndDescription(slackChannel) { - // @TODO implement model specific function - const roomUpdate = { - ts: new Date(slackChannel.created * 1000), - }; - - if (!_.isEmpty(slackChannel.topic && slackChannel.topic.value)) { - roomUpdate.topic = slackChannel.topic.value; - } - - if (!_.isEmpty(slackChannel.purpose && slackChannel.purpose.value)) { - roomUpdate.description = slackChannel.purpose.value; - } - - Rooms.update({ _id: slackChannel.rocketId }, { $set: roomUpdate, $addToSet: { importIds: slackChannel.id } }); - } - - _importMpims(startedByUserId, channelNames) { - const maxUsers = settings.get('DirectMesssage_maxUsers') || 1; - - - for (const list of this.mpimsLists) { - list.channels.forEach((channel) => { - if (!channel.do_import) { - this.addCountCompleted(1); - return; + if (isBotMessage) { + newMessage.bot = true; } - if (channelNames.includes(channel.name)) { - this.logger.warn(`Duplicated multi party IM name will be skipped: ${ channel.name }`); - return; + if (message.subtype === 'me_message') { + newMessage.msg = `_${ text }_`; + } else { + newMessage.msg = text; } - channelNames.push(channel.name); - - Meteor.runAsUser(startedByUserId, () => { - const users = this._getChannelUserList(channel, true, true); - const existingRoom = Rooms.findOneDirectRoomContainingAllUserIDs(users, { fields: { _id: 1 } }); + if (message.thread_ts) { + if (message.thread_ts === message.ts) { + if (message.reply_users) { + const replies = new Set(); + message.reply_users.forEach((item) => { + replies.add(this._replaceSlackUserId(item)); + }); - if (existingRoom) { - channel.rocketId = existingRoom._id; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } else { - const userId = this.getImportedRocketUserIdFromSlackUserId(channel.creator) || startedByUserId; - Meteor.runAsUser(userId, () => { - // If there are too many users for a direct room, then create a private group instead - if (users.length > maxUsers) { - const usernames = users.map((user) => user.username); - const group = Meteor.call('createPrivateGroup', channel.name, usernames); - channel.rocketId = group.rid; - return; + if (replies.length) { + newMessage.replies = Array.from(replies); } + } else if (message.replies) { + const replies = new Set(); + message.repĺies.forEach((item) => { + replies.add(this._replaceSlackUserId(item.user)); + }); - const newRoom = createDirectRoom(users); - channel.rocketId = newRoom._id; - }); - - this._updateImportedChannelTopicAndDescription(channel); - } - - this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - } - - _importDMs(startedByUserId, channelNames) { - for (const list of this.dmsLists) { - list.channels.forEach((channel) => { - if (channelNames.includes(channel.id)) { - this.logger.warn(`Duplicated DM id will be skipped (DMs): ${ channel.id }`); - return; - } - channelNames.push(channel.id); - - if (!channel.members || channel.members.length !== 2) { - this.addCountCompleted(1); - return; - } - - Meteor.runAsUser(startedByUserId, () => { - const user1 = this.getRocketUserFromUserId(channel.members[0]); - const user2 = this.getRocketUserFromUserId(channel.members[1]); - - const existingRoom = Rooms.findOneDirectRoomContainingAllUserIDs([user1, user2], { fields: { _id: 1 } }); - - if (existingRoom) { - channel.rocketId = existingRoom._id; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } else { - if (!user1) { - this.logger.error(`DM creation: User not found for id ${ channel.members[0] } and channel id ${ channel.id }`); - return; - } - - if (!user2) { - this.logger.error(`DM creation: User not found for id ${ channel.members[1] } and channel id ${ channel.id }`); - return; - } - - const roomInfo = Meteor.runAsUser(user1._id, () => Meteor.call('createDirectMessage', user2.username)); - channel.rocketId = roomInfo.rid; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } - - this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - } - - _importMessages(startedByUserId, channelNames) { - const missedTypes = {}; - super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - for (const channel of channelNames) { - if (!channel) { - continue; - } - - const slackChannel = this.getSlackChannelFromName(channel); - - const room = Rooms.findOneById(slackChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); - if (!room) { - this.logger.error(`ROOM not found: ${ channel }`); - continue; - } - const messagePacks = this.collection.find({ import: this.importRecord._id, type: 'messages', channel }); - - Meteor.runAsUser(startedByUserId, () => { - messagePacks.forEach((pack) => { - const packId = pack.i ? `${ pack.date }.${ pack.i }` : pack.date; - - this.updateRecord({ messagesstatus: `${ channel }/${ packId } (${ pack.messages.length })` }); - pack.messages.forEach((message) => { - try { - return this.performMessageImport(message, room, missedTypes, slackChannel); - } catch (e) { - this.logger.warn(`Failed to import message with timestamp ${ String(message.ts) } to room ${ room._id }`); - this.logger.debug(e); + if (replies.length) { + newMessage.replies = Array.from(replies); + } + } else { + this.logger.warn(`Failed to import the parent comment, message: ${ newMessage._id }. Missing replies/reply_users field`); } - }); - }); - }); - } - - if (!_.isEmpty(missedTypes)) { - console.log('Missed import types:', missedTypes); - } - } - _applyUserSelection(importSelection) { - if (importSelection.users.length === 3 && importSelection.users[0].user_id === 'users') { - const regularUsers = importSelection.users[0].do_import; - const botUsers = importSelection.users[1].do_import; - const deletedUsers = importSelection.users[2].do_import; - - for (const list of this.userLists) { - Object.keys(list.users).forEach((k) => { - const u = list.users[k]; - - if (u.is_bot) { - u.do_import = botUsers; - } else if (u.deleted) { - u.do_import = deletedUsers; + newMessage.tcount = message.reply_count; + newMessage.tlm = new Date(parseInt(message.latest_reply.split('.')[0]) * 1000); } else { - u.do_import = regularUsers; - } - }); - - this.collection.update({ _id: list._id }, { $set: { users: list.users } }); - } - } - - Object.keys(importSelection.users).forEach((key) => { - const user = importSelection.users[key]; - - for (const list of this.userLists) { - Object.keys(list.users).forEach((k) => { - const u = list.users[k]; - if (u.id === user.user_id) { - u.do_import = user.do_import; - } - }); - - this.collection.update({ _id: list._id }, { $set: { users: list.users } }); - } - }); - } - - _applyChannelSelection(importSelection) { - const iterateChannelList = (listList, channel_id, do_import) => { - for (const list of listList) { - for (const c of list.channels) { - if (!c) { - continue; - } - - if (channel_id === '*') { - if (!c.archived) { - c.do_import = do_import; - } - } else if (channel_id === '*/archived') { - if (c.archived) { - c.do_import = do_import; - } - } else if (c.id === channel_id) { - c.do_import = do_import; + newMessage.tmid = this.makeSlackMessageId(slackChannelId, message.thread_ts); } } - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - }; - - Object.keys(importSelection.channels).forEach((key) => { - const channel = importSelection.channels[key]; - - switch (channel.channel_id) { - case 'channels': - iterateChannelList(this.channelsLists, '*', channel.do_import); - break; - case 'archived_channels': - iterateChannelList(this.channelsLists, '*/archived', channel.do_import); - break; - case 'groups': - iterateChannelList(this.groupsLists, '*', channel.do_import); - break; - case 'archived_groups': - iterateChannelList(this.groupsLists, '*/archived', channel.do_import); - break; - case 'mpims': - iterateChannelList(this.mpimsLists, '*', channel.do_import); - break; - case 'archived_mpims': - iterateChannelList(this.mpimsLists, '*/archived', channel.do_import); - break; - default: - iterateChannelList(this.channelsLists, channel.channel_id, channel.do_import); - iterateChannelList(this.groupsLists, channel.channel_id, channel.do_import); - iterateChannelList(this.mpimsLists, channel.channel_id, channel.do_import); - break; - } - }); - } - - startImport(importSelection) { - const bots = this.collection.findOne({ import: this.importRecord._id, type: 'bots' }); - if (bots) { - this.bots = bots.bots || {}; - } else { - this.bots = {}; - } - - this.userLists = RawImports.find({ import: this.importRecord._id, type: 'users' }).fetch(); - this.channelsLists = RawImports.find({ import: this.importRecord._id, type: 'channels' }).fetch(); - this.groupsLists = RawImports.find({ import: this.importRecord._id, type: 'groups' }).fetch(); - this.dmsLists = RawImports.find({ import: this.importRecord._id, type: 'DMs' }).fetch(); - this.mpimsLists = RawImports.find({ import: this.importRecord._id, type: 'mpims' }).fetch(); - - this._userDataCache = {}; - this._anyExistingSlackMessage = Boolean(Messages.findOne({ _id: /slack\-.*/ })); - this.reloadCount(); - - super.startImport(importSelection); - const start = Date.now(); - - this._applyUserSelection(importSelection); - this._applyChannelSelection(importSelection); - - const channelNames = []; - - const startedByUserId = Meteor.userId(); - Meteor.defer(() => { - try { - this._importUsers(startedByUserId); - - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - this._importChannels(startedByUserId, channelNames); - this._importGroups(startedByUserId, channelNames); - this._importMpims(startedByUserId, channelNames); - - this._importDMs(startedByUserId, channelNames); - - this._importMessages(startedByUserId, channelNames); - - super.updateProgress(ProgressStep.FINISHING); - - try { - this._archiveChannelsAsNeeded(startedByUserId, this.channelsLists); - this._archiveChannelsAsNeeded(startedByUserId, this.groupsLists); - this._archiveChannelsAsNeeded(startedByUserId, this.mpimsLists); - - this._updateRoomsLastMessage(this.channelsLists); - this._updateRoomsLastMessage(this.groupsLists); - this._updateRoomsLastMessage(this.mpimsLists); - this._updateRoomsLastMessage(this.dmsLists); - } catch (e) { - // If it failed to archive some channel, it's no reason to flag the import as incomplete - // Just report the error but keep the import as successful. - console.error(e); + if (message.edited) { + newMessage.editedAt = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000); + if (message.edited.user) { + newMessage.editedBy = this._replaceSlackUserId(message.edited.user); + } } - super.updateProgress(ProgressStep.DONE); - - this.logger.log(`Import took ${ Date.now() - start } milliseconds.`); - } catch (e) { - this.logger.error(e); - super.updateProgress(ProgressStep.ERROR); - } - - this._userIdReference = {}; - }); - - return this.getProgress(); - } - _archiveChannelsAsNeeded(startedByUserId, listList) { - for (const list of listList) { - list.channels.forEach((channel) => { - if (channel.do_import && channel.is_archived && channel.rocketId) { - Meteor.runAsUser(startedByUserId, function() { - Meteor.call('archiveRoom', channel.rocketId); - }); + if (message.attachments) { + newMessage.attachments = this.convertMessageAttachments(message.attachments); } - }); - } - } - _updateRoomsLastMessage(listList) { - for (const list of listList) { - list.channels.forEach((channel) => { - if (channel.do_import && channel.rocketId) { - Rooms.resetLastMessageById(channel.rocketId); + if (message.icons && message.icons.emoji) { + newMessage.emoji = message.icons.emoji; } - }); - } - } - - getSlackChannelFromName(channelName) { - for (const list of this.channelsLists) { - const channel = list.channels.find((channel) => channel.name === channelName); - if (channel) { - return channel; - } - } - for (const list of this.groupsLists) { - const group = list.channels.find((channel) => channel.name === channelName); - if (group) { - return group; - } - } - - for (const list of this.mpimsLists) { - const group = list.channels.find((channel) => channel.name === channelName); - if (group) { - return group; - } - } - - for (const list of this.dmsLists) { - const dm = list.channels.find((channel) => channel.id === channelName); - if (dm) { - return dm; + this.parseMentions(newMessage); + this.converter.addMessage(newMessage, this._useUpsert); } } } - _getBasicUserData(userId) { - if (this._userDataCache[userId]) { - return this._userDataCache[userId]; - } - - this._userDataCache[userId] = Users.findOneById(userId, { fields: { username: 1, name: 1 } }); - return this._userDataCache[userId]; - } - - getRocketUserFromUserId(userId) { - if (userId === 'rocket.cat' || userId === 'USLACKBOT') { - return this._getBasicUserData('rocket.cat'); - } - - const rocketId = this._getUserRocketId(userId); - if (rocketId) { - return this._getBasicUserData(rocketId); - } - - if (userId in this.bots) { - return this._getBasicUserData('rocket.cat'); - } - } - - getImportedRocketUserIdFromSlackUserId(slackUserId) { - if (slackUserId.toUpperCase() === 'USLACKBOT') { + _replaceSlackUserId(userId) { + if (userId === 'USLACKBOT') { return 'rocket.cat'; } - for (const list of this.userLists) { - for (const user of list.users) { - if (user.id !== slackUserId) { - continue; - } - - if (user.do_import) { - return user.rocketId; - } + return userId; + } - if (user.is_bot) { - return 'rocket.cat'; - } - } - } + _replaceSlackUserIds(members) { + return members.map((userId) => this._replaceSlackUserId(userId)); } convertSlackMessageToRocketChat(message) { @@ -1162,11 +565,8 @@ export class SlackImporter extends Base { message = message.replace(/<(http[s]?:[^>|]*)>/g, '$1'); message = message.replace(/<(http[s]?:[^|]*)\|([^>]*)>/g, '[$2]($1)'); message = message.replace(/<#([^|]*)\|([^>]*)>/g, '#$2'); - - for (const userReplace of Array.from(this.userTags)) { - message = message.replace(userReplace.slack, userReplace.rocket); - message = message.replace(userReplace.slackLong, userReplace.rocket); - } + message = message.replace(/<@([^|]*)\|([^>]*)>/g, '@$1'); + message = message.replace(/<@([^|>]*)>/g, '@$1'); } else { message = ''; } diff --git a/app/importer-slack/server/index.js b/app/importer-slack/server/index.js index 44a1b3bab84c..d8499b517f00 100644 --- a/app/importer-slack/server/index.js +++ b/app/importer-slack/server/index.js @@ -1 +1,5 @@ -import './adder'; +import { SlackImporter } from './importer'; +import { Importers } from '../../importer/server'; +import { SlackImporterInfo } from '../lib/info'; + +Importers.add(new SlackImporterInfo(), SlackImporter); diff --git a/app/importer/server/classes/ImportDataConverter.ts b/app/importer/server/classes/ImportDataConverter.ts new file mode 100644 index 000000000000..b92fab40d1e0 --- /dev/null +++ b/app/importer/server/classes/ImportDataConverter.ts @@ -0,0 +1,841 @@ +import { Meteor } from 'meteor/meteor'; +import { Accounts } from 'meteor/accounts-base'; +import _ from 'underscore'; + +import { ImportData } from '../models/ImportData'; +import { IImportUser } from '../definitions/IImportUser'; +import { IImportMessage, IImportMessageReaction } from '../definitions/IImportMessage'; +import { IImportChannel } from '../definitions/IImportChannel'; +import { IImportUserRecord, IImportChannelRecord, IImportMessageRecord } from '../definitions/IImportRecord'; +import { Users, Rooms, Subscriptions } from '../../../models/server'; +import { generateUsernameSuggestion, insertMessage } from '../../../lib/server'; +import { setUserActiveStatus } from '../../../lib/server/functions/setUserActiveStatus'; +import { IUser } from '../../../../definition/IUser'; + +// @ts-ignore //@ToDo: Add the Logger class definitions. +type FakeLogger = Logger; + +type IRoom = Record; +type IMessage = Record; +type IUserIdentification = { + _id: string; + username: string | undefined; +}; +type IMentionedUser = { + _id: string; + username: string; + name?: string; +}; +type IMentionedChannel = { + _id: string; + name: string; +}; + +type IMessageReaction = { + name: string; + usernames: Array; +}; + +type IMessageReactions = Record; + +interface IConversionCallbacks { + beforeImportFn?: { + (data: IImportUser | IImportChannel | IImportMessage, type: string): boolean; + }; + afterImportFn?: { + (data: IImportUser | IImportChannel | IImportMessage, type: string): void; + }; +} + +const guessNameFromUsername = (username: string): string => + username + .replace(/\W/g, ' ') + .replace(/\s(.)/g, (u) => u.toUpperCase()) + .replace(/^(.)/, (u) => u.toLowerCase()) + .replace(/^\w/, (u) => u.toUpperCase()); + +export class ImportDataConverter { + private _userCache: Map; + + // display name uses a different cache because it's only used on mentions so we don't need to load it every time we load an user + private _userDisplayNameCache: Map; + + private _roomCache: Map; + + private _roomNameCache: Map; + + private _logger: FakeLogger; + + constructor() { + this._userCache = new Map(); + this._userDisplayNameCache = new Map(); + this._roomCache = new Map(); + this._roomNameCache = new Map(); + } + + setLogger(logger: FakeLogger): void { + this._logger = logger; + } + + addUserToCache(importId: string, _id: string, username: string | undefined): IUserIdentification { + const cache = { + _id, + username, + }; + + this._userCache.set(importId, cache); + return cache; + } + + addUserDisplayNameToCache(importId: string, name: string): string { + this._userDisplayNameCache.set(importId, name); + return name; + } + + addRoomToCache(importId: string, rid: string): string { + this._roomCache.set(importId, rid); + return rid; + } + + addRoomNameToCache(importId: string, name: string): string { + this._roomNameCache.set(importId, name); + return name; + } + + addUserDataToCache(userData: IImportUser): void { + if (!userData._id) { + return; + } + if (!userData.importIds.length) { + return; + } + + this.addUserToCache(userData.importIds[0], userData._id, userData.username); + } + + addObject(type: string, data: Record, options: Record = {}): void { + ImportData.model.rawCollection().insert({ + data, + dataType: type, + ...options, + }); + } + + addUser(data: IImportUser): void { + this.addObject('user', data); + } + + addChannel(data: IImportChannel): void { + this.addObject('channel', data); + } + + addMessage(data: IImportMessage, useQuickInsert = false): void { + this.addObject('message', data, { + useQuickInsert: useQuickInsert || undefined, + }); + } + + updateUserId(_id: string, userData: IImportUser): void { + const updateData: Record = { + $set: { + statusText: userData.statusText || undefined, + roles: userData.roles || ['user'], + type: userData.type || 'user', + bio: userData.bio || undefined, + name: userData.name || undefined, + }, + }; + + if (userData.importIds?.length) { + updateData.$addToSet = { + importIds: { + $each: userData.importIds, + }, + }; + } + + Users.update({ _id }, updateData); + } + + updateUser(existingUser: IUser, userData: IImportUser): void { + userData._id = existingUser._id; + + this.updateUserId(userData._id, userData); + + if (userData.importIds.length) { + this.addUserToCache(userData.importIds[0], existingUser._id, existingUser.username); + } + + if (userData.avatarUrl) { + try { + Users.update({ _id: existingUser._id }, { $set: { _pendingAvatarUrl: userData.avatarUrl } }); + } catch (error) { + this._logger.warn(`Failed to set ${ existingUser._id }'s avatar from url ${ userData.avatarUrl }`); + this._logger.error(error); + } + } + } + + insertUser(userData: IImportUser): IUser { + const password = `${ Date.now() }${ userData.name || '' }${ userData.emails.length ? userData.emails[0].toUpperCase() : '' }`; + const userId = userData.emails.length ? Accounts.createUser({ + email: userData.emails[0], + password, + }) : Accounts.createUser({ + username: userData.username, + password, + // @ts-ignore + joinDefaultChannelsSilenced: true, + }); + + userData._id = userId; + const user = Users.findOneById(userId, {}); + + if (user && userData.importIds.length) { + this.addUserToCache(userData.importIds[0], user._id, userData.username); + } + + Meteor.runAsUser(userId, () => { + Meteor.call('setUsername', userData.username, { joinDefaultChannelsSilenced: true }); + if (userData.name) { + Users.setName(userId, userData.name); + } + + this.updateUserId(userId, userData); + + if (userData.utcOffset) { + Users.setUtcOffset(userId, userData.utcOffset); + } + + if (userData.avatarUrl) { + try { + Users.update({ _id: userId }, { $set: { _pendingAvatarUrl: userData.avatarUrl } }); + } catch (error) { + this._logger.warn(`Failed to set ${ userId }'s avatar from url ${ userData.avatarUrl }`); + this._logger.error(error); + } + } + }); + + return user; + } + + convertUsers({ beforeImportFn, afterImportFn }: IConversionCallbacks = {}): void { + const users = ImportData.find({ dataType: 'user' }); + users.forEach(({ data, _id }: IImportUserRecord) => { + try { + if (beforeImportFn && !beforeImportFn(data, 'user')) { + this.skipRecord(_id); + return; + } + + data.emails = data.emails.filter((item) => item); + data.importIds = data.importIds.filter((item) => item); + + if (!data.emails.length && !data.username) { + throw new Error('importer-user-missing-email-and-username'); + } + + let existingUser; + if (data.emails.length) { + existingUser = Users.findOneByEmailAddress(data.emails[0], {}); + } + + if (data.username) { + // If we couldn't find one by their email address, try to find an existing user by their username + if (!existingUser) { + existingUser = Users.findOneByUsernameIgnoringCase(data.username, {}); + } + } else { + data.username = generateUsernameSuggestion({ + name: data.name, + emails: data.emails, + }); + } + + if (existingUser) { + this.updateUser(existingUser, data); + } else { + if (!data.name && data.username) { + data.name = guessNameFromUsername(data.username); + } + + existingUser = this.insertUser(data); + } + + // Deleted users are 'inactive' users in Rocket.Chat + if (data.deleted && existingUser?.active) { + setUserActiveStatus(data._id, false, true); + } + + if (afterImportFn) { + afterImportFn(data, 'user'); + } + } catch (e) { + this.saveError(_id, e); + } + }); + } + + saveNewId(importId: string, newId: string): void { + ImportData.update({ + _id: importId, + }, { + $set: { + id: newId, + }, + }); + } + + saveError(importId: string, error: Error): void { + this._logger.error(error); + ImportData.update({ + _id: importId, + }, { + $push: { + errors: { + message: error.message, + stack: error.stack, + }, + }, + }); + } + + skipRecord(_id: string): void { + ImportData.update({ + _id, + }, { + $set: { + skipped: true, + }, + }); + } + + convertMessageReactions(importedReactions: Record): undefined | IMessageReactions { + const reactions: IMessageReactions = {}; + + for (const name in importedReactions) { + if (!importedReactions.hasOwnProperty(name)) { + continue; + } + const { users } = importedReactions[name]; + + if (!users.length) { + continue; + } + + const reaction: IMessageReaction = { + name, + usernames: [], + }; + + for (const importId of users) { + const username = this.findImportedUsername(importId); + if (username && !reaction.usernames.includes(username)) { + reaction.usernames.push(username); + } + } + + if (reaction.usernames.length) { + reactions[name] = reaction; + } + } + + if (Object.keys(reactions).length > 0) { + return reactions; + } + } + + convertMessageReplies(replies: Array): Array { + const result: Array = []; + for (const importId of replies) { + const userId = this.findImportedUserId(importId); + if (userId && !result.includes(userId)) { + result.push(userId); + } + } + return result; + } + + convertMessageMentions(message: IImportMessage): Array | undefined { + const { mentions } = message; + if (!mentions) { + return undefined; + } + + const result: Array = []; + for (const importId of mentions) { + // eslint-disable-next-line no-extra-parens + if (importId === ('all' as 'string') || importId === 'here') { + result.push({ + _id: importId, + username: importId, + }); + continue; + } + + // Loading the name will also store the remaining data on the cache if it's missing, so this won't run two queries + const name = this.findImportedUserDisplayName(importId); + const data = this.findImportedUser(importId); + + if (!data) { + throw new Error('importer-message-mentioned-user-not-found'); + } + if (!data.username) { + throw new Error('importer-message-mentioned-username-not-found'); + } + + message.msg = message.msg.replace(new RegExp(`\@${ importId }`, 'gi'), `@${ data.username }`); + + result.push({ + _id: data._id, + username: data.username as 'string', + name, + }); + } + return result; + } + + convertMessageChannels(message: IImportMessage): Array | undefined { + const { channels } = message; + if (!channels) { + return; + } + + const result: Array = []; + for (const importId of channels) { + // loading the name will also store the id on the cache if it's missing, so this won't run two queries + const name = this.findImportedRoomName(importId); + const _id = this.findImportedRoomId(importId); + + if (!_id || !name) { + this._logger.warn(`Mentioned room not found: ${ importId }`); + continue; + } + + message.msg = message.msg.replace(new RegExp(`\#${ importId }`, 'gi'), `#${ name }`); + + result.push({ + _id, + name, + }); + } + + return result; + } + + convertMessages({ beforeImportFn, afterImportFn }: IConversionCallbacks = {}): void { + const rids: Array = []; + const messages = ImportData.find({ dataType: 'message' }); + messages.forEach(({ data: m, _id }: IImportMessageRecord) => { + try { + if (beforeImportFn && !beforeImportFn(m, 'message')) { + this.skipRecord(_id); + return; + } + + if (!m.ts || isNaN(m.ts as unknown as number)) { + throw new Error('importer-message-invalid-timestamp'); + } + + const creator = this.findImportedUser(m.u._id); + if (!creator) { + this._logger.warn(`Imported user not found: ${ m.u._id }`); + throw new Error('importer-message-unknown-user'); + } + + const rid = this.findImportedRoomId(m.rid); + if (!rid) { + throw new Error('importer-message-unknown-room'); + } + if (!rids.includes(rid)) { + rids.push(rid); + } + + // Convert the mentions and channels first because these conversions can also modify the msg in the message object + const mentions = m.mentions && this.convertMessageMentions(m); + const channels = m.channels && this.convertMessageChannels(m); + + const msgObj: IMessage = { + rid, + u: { + _id: creator._id, + username: creator.username, + }, + msg: m.msg, + ts: m.ts, + t: m.t || undefined, + groupable: m.groupable, + tmid: m.tmid, + tlm: m.tlm, + tcount: m.tcount, + replies: m.replies && this.convertMessageReplies(m.replies), + editedAt: m.editedAt, + editedBy: m.editedBy && (this.findImportedUser(m.editedBy) || undefined), + mentions, + channels, + _importFile: m._importFile, + url: m.url, + attachments: m.attachments, + bot: m.bot, + emoji: m.emoji, + alias: m.alias, + }; + + if (m._id) { + msgObj._id = m._id; + } + + if (m.reactions) { + msgObj.reactions = this.convertMessageReactions(m.reactions); + } + + try { + insertMessage(creator, msgObj, rid, true); + } catch (e) { + this._logger.warn(`Failed to import message with timestamp ${ String(msgObj.ts) } to room ${ rid }`); + this._logger.error(e); + } + + if (afterImportFn) { + afterImportFn(m, 'message'); + } + } catch (e) { + this.saveError(_id, e); + } + }); + + for (const rid of rids) { + try { + Rooms.resetLastMessageById(rid); + } catch (e) { + this._logger.warn(`Failed to update last message of room ${ rid }`); + this._logger.error(e); + } + } + } + + updateRoom(room: IRoom, roomData: IImportChannel, startedByUserId: string): void { + roomData._id = room._id; + + // eslint-disable-next-line no-extra-parens + if ((roomData._id as string).toUpperCase() === 'GENERAL' && roomData.name !== room.name) { + Meteor.runAsUser(startedByUserId, () => { + Meteor.call('saveRoomSettings', 'GENERAL', 'roomName', roomData.name); + }); + } + + this.updateRoomId(room._id, roomData); + } + + findDMForImportedUsers(...users: Array): IImportChannel | undefined { + const record = ImportData.findDMForImportedUsers(...users); + if (record) { + return record.data; + } + } + + findImportedRoomId(importId: string): string | null { + if (this._roomCache.has(importId)) { + return this._roomCache.get(importId) as string; + } + + const options = { + fields: { + _id: 1, + }, + }; + + const room = Rooms.findOneByImportId(importId, options); + if (room) { + return this.addRoomToCache(importId, room._id); + } + + return null; + } + + findImportedRoomName(importId: string): string | undefined { + if (this._roomNameCache.has(importId)) { + return this._roomNameCache.get(importId) as string; + } + + const options = { + fields: { + _id: 1, + name: 1, + }, + }; + + const room = Rooms.findOneByImportId(importId, options); + if (room) { + if (!this._roomCache.has(importId)) { + this.addRoomToCache(importId, room._id); + } + return this.addRoomNameToCache(importId, room.name); + } + } + + findImportedUser(importId: string): IUserIdentification | null { + const options = { + fields: { + _id: 1, + username: 1, + }, + }; + + if (importId === 'rocket.cat') { + return { + _id: 'rocket.cat', + username: 'rocket.cat', + }; + } + + if (this._userCache.has(importId)) { + return this._userCache.get(importId) as IUserIdentification; + } + + const user = Users.findOneByImportId(importId, options); + if (user) { + return this.addUserToCache(importId, user._id, user.username); + } + + return null; + } + + findImportedUserId(_id: string): string | undefined { + const data = this.findImportedUser(_id); + return data?._id; + } + + findImportedUsername(_id: string): string | undefined { + const data = this.findImportedUser(_id); + return data?.username; + } + + findImportedUserDisplayName(importId: string): string | undefined { + const options = { + fields: { + _id: 1, + name: 1, + username: 1, + }, + }; + + if (this._userDisplayNameCache.has(importId)) { + return this._userDisplayNameCache.get(importId); + } + + const user = importId === 'rocket.cat' ? Users.findOneById('rocket.cat', options) : Users.findOneByImportId(importId, options); + if (user) { + if (!this._userCache.has(importId)) { + this.addUserToCache(importId, user._id, user.username); + } + + return this.addUserDisplayNameToCache(importId, user.name); + } + } + + updateRoomId(_id: string, roomData: IImportChannel): void { + const set = { + ts: roomData.ts, + topic: roomData.topic, + description: roomData.description, + }; + + const roomUpdate: {$set?: Record; $addToSet?: Record} = {}; + + if (Object.keys(set).length > 0) { + roomUpdate.$set = set; + } + + if (roomData.importIds.length) { + roomUpdate.$addToSet = { + importIds: { + $each: roomData.importIds, + }, + }; + } + + if (roomUpdate.$set || roomUpdate.$addToSet) { + Rooms.update({ _id: roomData._id }, roomUpdate); + } + } + + getRoomCreatorId(roomData: IImportChannel, startedByUserId: string): string { + if (roomData.u) { + const creatorId = this.findImportedUserId(roomData.u._id); + if (creatorId) { + return creatorId; + } + + if (roomData.t !== 'd') { + return startedByUserId; + } + + throw new Error('importer-channel-invalid-creator'); + } + + if (roomData.t === 'd') { + for (const member of roomData.users) { + const userId = this.findImportedUserId(member); + if (userId) { + return userId; + } + } + } + + throw new Error('importer-channel-invalid-creator'); + } + + insertRoom(roomData: IImportChannel, startedByUserId: string): void { + // Find the rocketchatId of the user who created this channel + const creatorId = this.getRoomCreatorId(roomData, startedByUserId); + const members = this.convertImportedIdsToUsernames(roomData.users, roomData.t !== 'd' ? creatorId : undefined); + + if (roomData.t === 'd') { + if (members.length < roomData.users.length) { + this._logger.warn(`One or more imported users not found: ${ roomData.users }`); + throw new Error('importer-channel-missing-users'); + } + } + + // Create the channel + try { + Meteor.runAsUser(creatorId, () => { + const roomInfo = roomData.t === 'd' + ? Meteor.call('createDirectMessage', ...members) + : Meteor.call(roomData.t === 'p' ? 'createPrivateGroup' : 'createChannel', roomData.name, members); + + roomData._id = roomInfo.rid; + }); + } catch (e) { + this._logger.warn(roomData.name, members); + this._logger.error(e); + throw e; + } + + this.updateRoomId(roomData._id as 'string', roomData); + } + + convertImportedIdsToUsernames(importedIds: Array, idToRemove: string | undefined = undefined): Array { + return importedIds.map((user) => { + if (user === 'rocket.cat') { + return user; + } + + if (this._userCache.has(user)) { + const cache = this._userCache.get(user); + if (cache) { + return cache.username; + } + } + + const obj = Users.findOneByImportId(user, { fields: { _id: 1, username: 1 } }); + if (obj) { + this.addUserToCache(user, obj._id, obj.username); + + if (idToRemove && obj._id === idToRemove) { + return false; + } + + return obj.username; + } + + return false; + }).filter((user) => user); + } + + findExistingRoom(data: IImportChannel): IRoom { + if (data._id && data._id.toUpperCase() === 'GENERAL') { + const room = Rooms.findOneById('GENERAL', {}); + // Prevent the importer from trying to create a new general + if (!room) { + throw new Error('importer-channel-general-not-found'); + } + + return room; + } + + if (data.t === 'd') { + const users = this.convertImportedIdsToUsernames(data.users); + if (users.length !== data.users.length) { + throw new Error('importer-channel-missing-users'); + } + + return Rooms.findDirectRoomContainingAllUsernames(users, {}); + } + + return Rooms.findOneByNonValidatedName(data.name, {}); + } + + convertChannels(startedByUserId: string, { beforeImportFn, afterImportFn }: IConversionCallbacks = {}): void { + const channels = ImportData.find({ dataType: 'channel' }); + channels.forEach(({ data: c, _id }: IImportChannelRecord) => { + try { + if (beforeImportFn && !beforeImportFn(c, 'channel')) { + this.skipRecord(_id); + return; + } + + if (!c.name && c.t !== 'd') { + throw new Error('importer-channel-missing-name'); + } + + c.importIds = c.importIds.filter((item) => item); + c.users = _.uniq(c.users); + + if (!c.importIds.length) { + throw new Error('importer-channel-missing-import-id'); + } + + const existingRoom = this.findExistingRoom(c); + + if (existingRoom) { + this.updateRoom(existingRoom, c, startedByUserId); + } else { + this.insertRoom(c, startedByUserId); + } + + if (c.archived && c._id) { + this.archiveRoomById(c._id); + } + + if (afterImportFn) { + afterImportFn(c, 'channel'); + } + } catch (e) { + this.saveError(_id, e); + } + }); + } + + archiveRoomById(rid: string): void { + Rooms.archiveById(rid); + Subscriptions.archiveByRoomId(rid); + } + + convertData(startedByUserId: string, callbacks: IConversionCallbacks = {}): void { + this.convertUsers(callbacks); + this.convertChannels(startedByUserId, callbacks); + this.convertMessages(callbacks); + + Meteor.defer(() => { + this.clearSuccessfullyImportedData(); + }); + } + + clearImportData(): void { + const rawCollection = ImportData.model.rawCollection(); + const remove = Meteor.wrapAsync(rawCollection.remove, rawCollection); + + remove({}); + } + + clearSuccessfullyImportedData(): void { + ImportData.model.rawCollection().remove({ + errors: { + $exists: false, + }, + }); + } +} diff --git a/app/importer/server/classes/ImporterBase.js b/app/importer/server/classes/ImporterBase.js index df2052eb8d6d..decfd093543f 100644 --- a/app/importer/server/classes/ImporterBase.js +++ b/app/importer/server/classes/ImporterBase.js @@ -7,67 +7,25 @@ import AdmZip from 'adm-zip'; import getFileType from 'file-type'; import { Progress } from './ImporterProgress'; -import { Selection } from './ImporterSelection'; import { ImporterWebsocket } from './ImporterWebsocket'; import { ProgressStep } from '../../lib/ImporterProgressStep'; import { ImporterInfo } from '../../lib/ImporterInfo'; import { RawImports } from '../models/RawImports'; import { Settings, Imports } from '../../../models'; import { Logger } from '../../../logger'; -import { FileUpload } from '../../../file-upload'; -import { sendMessage } from '../../../lib'; +import { ImportDataConverter } from './ImportDataConverter'; +import { ImportData } from '../models/ImportData'; +import { t } from '../../../utils/server'; +import { + Selection, + SelectionChannel, + SelectionUser, +} from '..'; /** * Base class for all of the importers. */ export class Base { - /** - * The max BSON object size we can store in MongoDB is 16777216 bytes - * but for some reason the mongo instanace which comes with Meteor - * errors out for anything close to that size. So, we are rounding it - * down to 8000000 bytes. - * - * @param {any} item The item to calculate the BSON size of. - * @returns {number} The size of the item passed in. - * @static - */ - static getBSONSize(item) { - const { calculateObjectSize } = require('bson'); - - return calculateObjectSize(item); - } - - /** - * The max BSON object size we can store in MongoDB is 16777216 bytes - * but for some reason the mongo instanace which comes with Meteor - * errors out for anything close to that size. So, we are rounding it - * down to 6000000 bytes. - * - * @returns {number} 8000000 bytes. - */ - static getMaxBSONSize() { - return 6000000; - } - - /** - * Splits the passed in array to at least one array which has a size that - * is safe to store in the database. - * - * @param {any[]} theArray The array to split out - * @returns {any[][]} The safe sized arrays - * @static - */ - static getBSONSafeArraysFromAnArray(theArray) { - const BSONSize = Base.getBSONSize(theArray); - const maxSize = Math.floor(theArray.length / Math.ceil(BSONSize / Base.getMaxBSONSize())); - const safeArrays = []; - let i = 0; - while (i < theArray.length) { - safeArrays.push(theArray.slice(i, i += maxSize)); - } - return safeArrays; - } - /** * Constructs a new importer, adding an empty collection, AdmZip property, and empty users & channels * @@ -84,6 +42,7 @@ export class Base { this.https = https; this.AdmZip = AdmZip; this.getFileType = getFileType; + this.converter = new ImportDataConverter(); this.prepare = this.prepare.bind(this); this.startImport = this.startImport.bind(this); @@ -92,11 +51,12 @@ export class Base { this.addCountToTotal = this.addCountToTotal.bind(this); this.addCountCompleted = this.addCountCompleted.bind(this); this.updateRecord = this.updateRecord.bind(this); - this.uploadFile = this.uploadFile.bind(this); this.info = info; this.logger = new Logger(`${ this.info.name } Importer`, {}); + this.converter.setLogger(this.logger); + this.progress = new Progress(this.info.key, this.info.name); this.collection = RawImports; @@ -196,7 +156,70 @@ export class Base { throw new Error(`Channels in the selected data wasn't found, it must but at least an empty array for the ${ this.info.name } importer.`); } - return this.updateProgress(ProgressStep.IMPORTING_STARTED); + this.updateProgress(ProgressStep.IMPORTING_STARTED); + this.reloadCount(); + const started = Date.now(); + const startedByUserId = Meteor.userId(); + + const beforeImportFn = (data, type) => { + switch (type) { + case 'channel': { + const id = data.t === 'd' ? '__directMessages__' : data.importIds[0]; + for (const channel of importSelection.channels) { + if (channel.channel_id === id) { + return channel.do_import; + } + } + + return false; + } + case 'user': { + const id = data.importIds[0]; + for (const user of importSelection.users) { + if (user.user_id === id) { + return user.do_import; + } + } + + return false; + } + } + + return true; + }; + + const afterImportFn = () => { + this.addCountCompleted(1); + }; + + Meteor.defer(() => { + try { + this.updateProgress(ProgressStep.IMPORTING_USERS); + this.converter.convertUsers({ beforeImportFn, afterImportFn }); + + this.updateProgress(ProgressStep.IMPORTING_CHANNELS); + this.converter.convertChannels(startedByUserId, { beforeImportFn, afterImportFn }); + + this.updateProgress(ProgressStep.IMPORTING_MESSAGES); + this.converter.convertMessages({ afterImportFn }); + + this.updateProgress(ProgressStep.FINISHING); + + Meteor.defer(() => { + this.converter.clearSuccessfullyImportedData(); + }); + + this.updateProgress(ProgressStep.DONE); + } catch (e) { + this.logger.error(e); + this.updateProgress(ProgressStep.ERROR); + } + + const timeTook = Date.now() - started; + this.logger.log(`Import took ${ timeTook } milliseconds.`); + }); + + return this.getProgress(); } /** @@ -352,18 +375,6 @@ export class Base { }); } - flagConflictingEmails(emailList) { - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.users.email': { $in: emailList }, - }, { - $set: { - 'fileData.users.$.is_email_taken': true, - 'fileData.users.$.do_import': false, - }, - }); - } - /** * Updates the import record with the given fields being `set`. * @@ -377,79 +388,23 @@ export class Base { return this.importRecord; } - /** - * Uploads the file to the storage. - * - * @param {any} details An object with details about the upload: `name`, `size`, `type`, and `rid`. - * @param {string} fileUrl Url of the file to download/import. - * @param {any} user The Rocket.Chat user. - * @param {any} room The Rocket.Chat Room. - * @param {Date} timeStamp The timestamp the file was uploaded - */ - uploadFile(details, fileUrl, user, room, timeStamp) { - this.logger.debug(`Uploading the file ${ details.name } from ${ fileUrl }.`); - const requestModule = /https/i.test(fileUrl) ? this.https : this.http; - - const fileStore = FileUpload.getStore('Uploads'); + buildSelection() { + this.updateProgress(ProgressStep.USER_SELECTION); - return requestModule.get(fileUrl, Meteor.bindEnvironment(function(res) { - const contentType = res.headers['content-type']; - if (!details.type && contentType) { - details.type = contentType; - } + const users = ImportData.getAllUsersForSelection(); + const channels = ImportData.getAllChannelsForSelection(); + const hasDM = ImportData.checkIfDirectMessagesExists(); - const rawData = []; - res.on('data', (chunk) => rawData.push(chunk)); - res.on('end', Meteor.bindEnvironment(() => { - fileStore.insert(details, Buffer.concat(rawData), function(err, file) { - if (err) { - throw new Error(err); - } else { - const url = FileUpload.getPath(`${ file._id }/${ encodeURI(file.name) }`); - - const attachment = { - title: file.name, - title_link: url, - }; - - if (/^image\/.+/.test(file.type)) { - attachment.image_url = url; - attachment.image_type = file.type; - attachment.image_size = file.size; - attachment.image_dimensions = file.identify != null ? file.identify.size : undefined; - } + const selectionUsers = users.map((u) => new SelectionUser(u.data.importIds[0], u.data.username, u.data.emails[0], Boolean(u.data.deleted), u.data.type === 'bot', true)); + const selectionChannels = channels.map((c) => new SelectionChannel(c.data.importIds[0], c.data.name, Boolean(c.data.archived), true, c.data.t === 'p', undefined, c.data.t === 'd')); + const selectionMessages = ImportData.countMessages(); - if (/^audio\/.+/.test(file.type)) { - attachment.audio_url = url; - attachment.audio_type = file.type; - attachment.audio_size = file.size; - } + if (hasDM) { + selectionChannels.push(new SelectionChannel('__directMessages__', t('Direct_Messages'), false, true, true, undefined, true)); + } - if (/^video\/.+/.test(file.type)) { - attachment.video_url = url; - attachment.video_type = file.type; - attachment.video_size = file.size; - } + const results = new Selection(this.name, selectionUsers, selectionChannels, selectionMessages); - const msg = { - rid: details.rid, - ts: timeStamp, - msg: '', - file: { - _id: file._id, - }, - groupable: false, - attachments: [attachment], - }; - - if ((details.message_id != null) && (typeof details.message_id === 'string')) { - msg._id = details.message_id; - } - - return sendMessage(user, msg, room, true); - } - }); - })); - })); + return results; } } diff --git a/app/importer/server/definitions/IImportChannel.ts b/app/importer/server/definitions/IImportChannel.ts new file mode 100644 index 000000000000..5ad24ed99afb --- /dev/null +++ b/app/importer/server/definitions/IImportChannel.ts @@ -0,0 +1,14 @@ +export interface IImportChannel { + _id?: string; + u?: { + _id: string; + }; + name?: string; + users: Array; + importIds: Array; + t: string; + topic?: string; + description?: string; + ts?: Date; + archived?: boolean; +} diff --git a/app/importer/server/definitions/IImportMessage.ts b/app/importer/server/definitions/IImportMessage.ts new file mode 100644 index 000000000000..ae5357bec482 --- /dev/null +++ b/app/importer/server/definitions/IImportMessage.ts @@ -0,0 +1,53 @@ +export type IImportedId = 'string'; + +export interface IImportMessageReaction { + name: string; + users: Array; +} + +export interface IImportPendingFile { + downloadUrl: string; + id: string; + size: number; + name: string; + external: boolean; + source: string; + original: Record; +} + +export interface IImportAttachment extends Record { + text: string; + title: string; + fallback: string; +} + +export interface IImportMessage { + _id?: IImportedId; + + rid: IImportedId; + u: { + _id: IImportedId; + }; + + msg: string; + alias?: string; + ts: Date; + t?: string; + reactions?: Record; + groupable?: boolean; + + tmid?: IImportedId; + tlm?: Date; + tcount?: number; + replies?: Array; + editedAt?: Date; + editedBy?: IImportedId; + mentions?: Array; + channels?: Array; + attachments?: IImportAttachment; + bot?: boolean; + emoji?: string; + + url?: string; + _importFile?: IImportPendingFile; +} diff --git a/app/importer/server/definitions/IImportRecord.ts b/app/importer/server/definitions/IImportRecord.ts new file mode 100644 index 000000000000..09d3a9418ef4 --- /dev/null +++ b/app/importer/server/definitions/IImportRecord.ts @@ -0,0 +1,28 @@ +import { IImportUser } from './IImportUser'; +import { IImportChannel } from './IImportChannel'; +import { IImportMessage } from './IImportMessage'; + +export interface IImportRecord { + data: IImportUser | IImportChannel | IImportMessage; + dataType: 'user' | 'channel' | 'message'; + _id: string; + options?: {}; +} + +export interface IImportUserRecord extends IImportRecord { + data: IImportUser; + dataType: 'user'; +} + +export interface IImportChannelRecord extends IImportRecord { + data: IImportChannel; + dataType: 'channel'; +} + +export interface IImportMessageRecord extends IImportRecord { + data: IImportMessage; + dataType: 'message'; + options: { + useQuickInsert?: boolean; + }; +} diff --git a/app/importer/server/definitions/IImportUser.ts b/app/importer/server/definitions/IImportUser.ts new file mode 100644 index 000000000000..6462cb054a9e --- /dev/null +++ b/app/importer/server/definitions/IImportUser.ts @@ -0,0 +1,17 @@ +export interface IImportUser { + // #ToDo: Remove this _id, as it isn't part of the imported data + _id?: string; + + username?: string; + emails: Array; + importIds: Array; + name?: string; + utcOffset?: number; + active?: boolean; + avatarUrl?: string; + deleted?: boolean; + statusText?: string; + roles?: Array; + type: 'user' | 'bot'; + bio?: string; +} diff --git a/app/importer/server/index.js b/app/importer/server/index.js index 0fed91e5b660..dd9e89ba0209 100644 --- a/app/importer/server/index.js +++ b/app/importer/server/index.js @@ -2,6 +2,7 @@ import { Base } from './classes/ImporterBase'; import { ImporterWebsocket } from './classes/ImporterWebsocket'; import { Progress } from './classes/ImporterProgress'; import { RawImports } from './models/RawImports'; +import { ImportData } from './models/ImportData'; import { Selection } from './classes/ImporterSelection'; import { SelectionChannel } from './classes/ImporterSelectionChannel'; import { SelectionUser } from './classes/ImporterSelectionUser'; @@ -25,6 +26,7 @@ export { Progress, ProgressStep, RawImports, + ImportData, Selection, SelectionChannel, SelectionUser, diff --git a/app/importer/server/methods/getImportFileData.js b/app/importer/server/methods/getImportFileData.js index f1029b9869f1..e49f2a903dd7 100644 --- a/app/importer/server/methods/getImportFileData.js +++ b/app/importer/server/methods/getImportFileData.js @@ -58,31 +58,17 @@ Meteor.methods({ ]; if (readySteps.indexOf(importer.instance.progress.step) >= 0) { - if (importer.instance.importRecord && importer.instance.importRecord.fileData) { - return importer.instance.importRecord.fileData; - } + return importer.instance.buildSelection(); } const fileName = importer.instance.importRecord.file; const fullFilePath = fs.existsSync(fileName) ? fileName : path.join(RocketChatImportFileInstance.absolutePath, fileName); - const results = importer.instance.prepareUsingLocalFile(fullFilePath); - - if (results instanceof Promise) { - return results.then((data) => { - importer.instance.updateRecord({ - fileData: data, - }); - - return data; - }).catch((e) => { - console.error(e); - throw new Meteor.Error(e); - }); + const promise = importer.instance.prepareUsingLocalFile(fullFilePath); + + if (promise && promise instanceof Promise) { + Promise.await(promise); } - importer.instance.updateRecord({ - fileData: results, - }); - return results; + return importer.instance.buildSelection(); }, }); diff --git a/app/importer/server/models/ImportData.ts b/app/importer/server/models/ImportData.ts new file mode 100644 index 000000000000..a6afb291e19c --- /dev/null +++ b/app/importer/server/models/ImportData.ts @@ -0,0 +1,88 @@ +import { Base } from '../../../models/server'; +import { IImportUserRecord, IImportChannelRecord } from '../definitions/IImportRecord'; + +class ImportDataModel extends Base { + constructor() { + super('import_data'); + } + + getAllUsersForSelection(): Array { + return this.find({ + dataType: 'user', + }, { + fields: { + 'data.importIds': 1, + 'data.username': 1, + 'data.emails': 1, + 'data.deleted': 1, + 'data.type': 1, + }, + }).fetch(); + } + + getAllChannelsForSelection(): Array { + return this.find({ + dataType: 'channel', + 'data.t': { + $ne: 'd', + }, + }, { + fields: { + 'data.importIds': 1, + 'data.name': 1, + 'data.archived': 1, + 'data.t': 1, + }, + }).fetch(); + } + + checkIfDirectMessagesExists(): boolean { + return this.find({ + dataType: 'channel', + 'data.t': 'd', + }, { + fields: { + _id: 1, + }, + }).count() > 0; + } + + countMessages(): number { + return this.find({ + dataType: 'message', + }).count(); + } + + findChannelImportIdByNameOrImportId(channelIdentifier: string): string | undefined { + const channel = this.findOne({ + dataType: 'channel', + $or: [ + { + 'data.name': channelIdentifier, + }, + { + 'data.importIds': channelIdentifier, + }, + ], + }, { + fields: { + 'data.importIds': 1, + }, + }); + + return channel?.data?.importIds?.shift(); + } + + findDMForImportedUsers(...users: Array): IImportChannelRecord | undefined { + const query = { + dataType: 'channel', + 'data.users': { + $all: users, + }, + }; + + return this.findOne(query); + } +} + +export const ImportData = new ImportDataModel(); diff --git a/app/importer/server/startup/setImportsToInvalid.js b/app/importer/server/startup/setImportsToInvalid.js index c5303516cb59..a7ec86373222 100644 --- a/app/importer/server/startup/setImportsToInvalid.js +++ b/app/importer/server/startup/setImportsToInvalid.js @@ -8,7 +8,7 @@ function runDrop(fn) { try { fn(); } catch (e) { - console.log('errror', e); // TODO: Remove + console.log('error', e); // TODO: Remove // ignored } } @@ -21,9 +21,7 @@ Meteor.startup(function() { // And there's still data for it on the temp collection // Then we can keep the data there to let the user try again if (lastOperation && [ProgressStep.USER_SELECTION, ProgressStep.ERROR].includes(lastOperation.status)) { - if (RawImports.find({ import: lastOperation._id }).count() > 0) { - idToKeep = lastOperation._id; - } + idToKeep = lastOperation._id; } if (idToKeep) { diff --git a/app/ldap/server/ldap.js b/app/ldap/server/ldap.js index 0dceec62d886..eafdd3161796 100644 --- a/app/ldap/server/ldap.js +++ b/app/ldap/server/ldap.js @@ -370,6 +370,14 @@ export default class LDAP { values[key] = value; } } + + if (key === 'ou' && Array.isArray(value)) { + value.forEach((item, index) => { + if (item instanceof Buffer) { + value[index] = item.toString(); + } + }); + } }); return values; diff --git a/app/ldap/server/sync.js b/app/ldap/server/sync.js index 9993fe54549e..1ef91246142e 100644 --- a/app/ldap/server/sync.js +++ b/app/ldap/server/sync.js @@ -320,12 +320,17 @@ export function mapLDAPGroupsToChannels(ldap, ldapUser, user) { for (const channel of channels) { let room = Rooms.findOneByNonValidatedName(channel); + if (!room) { room = createRoomForSync(channel); } if (isUserInLDAPGroup(ldap, ldapUser, user, ldapField)) { - userChannels.push(room._id); - } else if (syncUserRolesEnforceAutoChannels) { + if (room.teamMain) { + logger.error(`Can't add user to channel ${ channel } because it is a team.`); + } else { + userChannels.push(room._id); + } + } else if (syncUserRolesEnforceAutoChannels && !room.teamMain) { const subscription = Subscriptions.findOneByRoomIdAndUserId(room._id, user._id); if (subscription) { removeUserFromRoom(room._id, user); diff --git a/app/lib/server/functions/checkUsernameAvailability.js b/app/lib/server/functions/checkUsernameAvailability.js index 0e7c3ca603e2..b197ad0b8780 100644 --- a/app/lib/server/functions/checkUsernameAvailability.js +++ b/app/lib/server/functions/checkUsernameAvailability.js @@ -4,6 +4,7 @@ import s from 'underscore.string'; import { escapeRegExp } from '../../../../lib/escapeRegExp'; import { settings } from '../../../settings'; import { Team } from '../../../../server/sdk'; +import { validateName } from './validateName'; let usernameBlackList = []; @@ -17,7 +18,7 @@ const usernameIsBlocked = (username, usernameBlackList) => usernameBlackList.len && usernameBlackList.some((restrictedUsername) => restrictedUsername.test(s.trim(escapeRegExp(username)))); export const checkUsernameAvailability = function(username) { - if (usernameIsBlocked(username, usernameBlackList)) { + if (usernameIsBlocked(username, usernameBlackList) || !validateName(username)) { return false; } diff --git a/app/lib/server/functions/index.js b/app/lib/server/functions/index.js index ecb2881083d2..df55892acca7 100644 --- a/app/lib/server/functions/index.js +++ b/app/lib/server/functions/index.js @@ -33,4 +33,5 @@ export { _setUsername, setUsername } from './setUsername'; export { unarchiveRoom } from './unarchiveRoom'; export { updateMessage } from './updateMessage'; export { validateCustomFields } from './validateCustomFields'; +export { validateName } from './validateName'; export { getAvatarSuggestionForUser } from './getAvatarSuggestionForUser'; diff --git a/app/lib/server/functions/insertMessage.js b/app/lib/server/functions/insertMessage.js index 6b0c7ea8a70b..560452250de7 100644 --- a/app/lib/server/functions/insertMessage.js +++ b/app/lib/server/functions/insertMessage.js @@ -76,8 +76,8 @@ const validateAttachment = (attachment) => { const validateBodyAttachments = (attachments) => attachments.map(validateAttachment); -export const insertMessage = function(user, message, room, upsert = false) { - if (!user || !message || !room._id) { +export const insertMessage = function(user, message, rid, upsert = false) { + if (!user || !message || !rid) { return false; } @@ -104,7 +104,7 @@ export const insertMessage = function(user, message, room, upsert = false) { _id, username, }; - message.rid = room._id; + message.rid = rid; if (!Match.test(message.msg, String)) { message.msg = ''; diff --git a/app/lib/server/functions/loadMessageHistory.js b/app/lib/server/functions/loadMessageHistory.js index 8e381a3fadc1..991f0976cc3e 100644 --- a/app/lib/server/functions/loadMessageHistory.js +++ b/app/lib/server/functions/loadMessageHistory.js @@ -10,7 +10,7 @@ settings.get('Hide_System_Messages', function(key, values) { hiddenTypes.forEach((item) => hideMessagesOfTypeServer.add(item)); }); -export const loadMessageHistory = function loadMessageHistory({ userId, rid, end, limit = 20, ls }) { +export const loadMessageHistory = function loadMessageHistory({ userId, rid, end, limit = 20, ls, showThreadMessages = true }) { const room = Rooms.findOne(rid, { fields: { sysMes: 1 } }); const hiddenMessageTypes = Array.isArray(room && room.sysMes) ? room.sysMes : Array.from(hideMessagesOfTypeServer.values()); // TODO probably remove on chained event system @@ -27,7 +27,20 @@ export const loadMessageHistory = function loadMessageHistory({ userId, rid, end }; } - const records = end != null ? Messages.findVisibleByRoomIdBeforeTimestampNotContainingTypes(rid, end, hiddenMessageTypes, options).fetch() : Messages.findVisibleByRoomIdNotContainingTypes(rid, hiddenMessageTypes, options).fetch(); + const records = end != null + ? Messages.findVisibleByRoomIdBeforeTimestampNotContainingTypes( + rid, + end, + hiddenMessageTypes, + options, + showThreadMessages, + ).fetch() + : Messages.findVisibleByRoomIdNotContainingTypes( + rid, + hiddenMessageTypes, + options, + showThreadMessages, + ).fetch(); const messages = normalizeMessagesForUser(records, userId); let unreadNotLoaded = 0; let firstUnread; @@ -37,12 +50,19 @@ export const loadMessageHistory = function loadMessageHistory({ userId, rid, end if ((firstMessage != null ? firstMessage.ts : undefined) > ls) { delete options.limit; - const unreadMessages = Messages.findVisibleByRoomIdBetweenTimestampsNotContainingTypes(rid, ls, firstMessage.ts, hiddenMessageTypes, { - limit: 1, - sort: { - ts: 1, + const unreadMessages = Messages.findVisibleByRoomIdBetweenTimestampsNotContainingTypes( + rid, + ls, + firstMessage.ts, + hiddenMessageTypes, + { + limit: 1, + sort: { + ts: 1, + }, }, - }); + showThreadMessages, + ); firstUnread = unreadMessages.fetch()[0]; unreadNotLoaded = unreadMessages.count(); diff --git a/app/lib/server/functions/parseUrlsInMessage.js b/app/lib/server/functions/parseUrlsInMessage.js new file mode 100644 index 000000000000..3c6b662f8181 --- /dev/null +++ b/app/lib/server/functions/parseUrlsInMessage.js @@ -0,0 +1,22 @@ +import { Markdown } from '../../../markdown/server'; + +export const parseUrlsInMessage = (message) => { + if (message.parseUrls === false) { + return message; + } + + message.html = message.msg; + message = Markdown.code(message); + + const urls = message.html.match(/([A-Za-z]{3,9}):\/\/([-;:&=\+\$,\w]+@{1})?([-A-Za-z0-9\.]+)+:?(\d+)?((\/[-\+=!:~%\/\.@\,\w]*)?\??([-\+=&!:;%@\/\.\,\w]+)?(?:#([^\s\)]+))?)?/g) || []; + if (urls) { + message.urls = urls.map((url) => ({ url })); + } + + message = Markdown.mountTokensBack(message, false); + message.msg = message.html; + delete message.html; + delete message.tokens; + + return message; +}; diff --git a/app/lib/server/functions/saveUserIdentity.js b/app/lib/server/functions/saveUserIdentity.js index 929b2fb3c81a..a4402e0127bd 100644 --- a/app/lib/server/functions/saveUserIdentity.js +++ b/app/lib/server/functions/saveUserIdentity.js @@ -3,6 +3,7 @@ import { setRealName } from './setRealName'; import { Messages, Rooms, Subscriptions, LivechatDepartmentAgents, Users } from '../../../models/server'; import { FileUpload } from '../../../file-upload/server'; import { updateGroupDMsName } from './updateGroupDMsName'; +import { validateName } from './validateName'; /** * @@ -25,6 +26,10 @@ export function saveUserIdentity(userId, { _id, name: rawName, username: rawUser const usernameChanged = previousUsername !== username; if (typeof rawUsername !== 'undefined' && usernameChanged) { + if (!validateName(username)) { + return false; + } + if (!setUsername(_id, username, user)) { return false; } diff --git a/app/lib/server/functions/sendMessage.js b/app/lib/server/functions/sendMessage.js index 593034e74de4..30f54e415f31 100644 --- a/app/lib/server/functions/sendMessage.js +++ b/app/lib/server/functions/sendMessage.js @@ -4,9 +4,9 @@ import { settings } from '../../../settings'; import { callbacks } from '../../../callbacks'; import { Messages } from '../../../models'; import { Apps } from '../../../apps/server'; -import { Markdown } from '../../../markdown/server'; import { isURL, isRelativeURL } from '../../../utils/lib/isURL'; import { FileUpload } from '../../../file-upload/server'; +import { parseUrlsInMessage } from './parseUrlsInMessage'; /** * IMPORTANT @@ -203,20 +203,7 @@ export const sendMessage = function(user, message, room, upsert = false) { } } - if (message.parseUrls !== false) { - message.html = message.msg; - message = Markdown.code(message); - - const urls = message.html.match(/([A-Za-z]{3,9}):\/\/([-;:&=\+\$,\w]+@{1})?([-A-Za-z0-9\.]+)+:?(\d+)?((\/[-\+=!:~%\/\.@\,\(\)\w]*)?\??([-\+=&!:;%@\/\.\,\w]+)?(?:#([^\s\)]+))?)?/g); - if (urls) { - message.urls = urls.map((url) => ({ url })); - } - - message = Markdown.mountTokensBack(message, false); - message.msg = message.html; - delete message.html; - delete message.tokens; - } + parseUrlsInMessage(message); message = callbacks.run('beforeSaveMessage', message, room); if (!settings.get('Livechat_kill_switch') || room.lastMessage.msg !== settings.get('Livechat_kill_switch_message')) { diff --git a/app/lib/server/functions/setUserActiveStatus.js b/app/lib/server/functions/setUserActiveStatus.js index c85651250715..fe9a889e0cf1 100644 --- a/app/lib/server/functions/setUserActiveStatus.js +++ b/app/lib/server/functions/setUserActiveStatus.js @@ -3,12 +3,31 @@ import { check } from 'meteor/check'; import { Accounts } from 'meteor/accounts-base'; import * as Mailer from '../../../mailer'; -import { Users, Subscriptions } from '../../../models'; +import { Users, Subscriptions, Rooms } from '../../../models'; import { settings } from '../../../settings'; import { relinquishRoomOwnerships } from './relinquishRoomOwnerships'; import { shouldRemoveOrChangeOwner, getSubscribedRoomsForUserWithDetails } from './getRoomsWithSingleOwner'; import { getUserSingleOwnedRooms } from './getUserSingleOwnedRooms'; +function reactivateDirectConversations(userId) { + // since both users can be deactivated at the same time, we should just reactivate rooms if both users are active + // for that, we need to fetch the direct messages, fetch the users involved and then the ids of rooms we can reactivate + const directConversations = Rooms.getDirectConversationsByUserId(userId, { projection: { _id: 1, uids: 1 } }).fetch(); + const userIds = directConversations.reduce((acc, r) => acc.push(...r.uids) && acc, []); + const uniqueUserIds = [...new Set(userIds)]; + const activeUsers = Users.findActiveByUserIds(uniqueUserIds, { projection: { _id: 1 } }).fetch(); + const activeUserIds = activeUsers.map((u) => u._id); + const roomsToReactivate = directConversations.reduce((acc, room) => { + const otherUserId = room.uids.find((u) => u !== userId); + if (activeUserIds.includes(otherUserId)) { + acc.push(room._id); + } + return acc; + }, []); + + Rooms.setDmReadOnlyByUserId(userId, roomsToReactivate, false, false); +} + export function setUserActiveStatus(userId, active, confirmRelinquish = false) { check(userId, String); check(active, Boolean); @@ -39,8 +58,10 @@ export function setUserActiveStatus(userId, active, confirmRelinquish = false) { if (active === false) { Users.unsetLoginTokens(userId); + Rooms.setDmReadOnlyByUserId(userId, undefined, true, false); } else { Users.unsetReason(userId); + reactivateDirectConversations(userId); } if (active && !settings.get('Accounts_Send_Email_When_Activating')) { return true; diff --git a/app/lib/server/functions/updateMessage.js b/app/lib/server/functions/updateMessage.js index 5dd785c9ab3f..950dbe53dff4 100644 --- a/app/lib/server/functions/updateMessage.js +++ b/app/lib/server/functions/updateMessage.js @@ -4,6 +4,7 @@ import { Messages, Rooms } from '../../../models'; import { settings } from '../../../settings'; import { callbacks } from '../../../callbacks'; import { Apps } from '../../../apps/server'; +import { parseUrlsInMessage } from './parseUrlsInMessage'; export const updateMessage = function(message, user, originalMessage) { if (!originalMessage) { @@ -39,8 +40,7 @@ export const updateMessage = function(message, user, originalMessage) { username: user.username, }; - const urls = message.msg.match(/([A-Za-z]{3,9}):\/\/([-;:&=\+\$,\w]+@{1})?([-A-Za-z0-9\.]+)+:?(\d+)?((\/[-\+=!:~%\/\.@\,\w]*)?\??([-\+=&!:;%@\/\.\,\w]+)?(?:#([^\s\)]+))?)?/g) || []; - message.urls = urls.map((url) => ({ url })); + parseUrlsInMessage(message); message = callbacks.run('beforeSaveMessage', message); diff --git a/app/lib/server/functions/validateName.ts b/app/lib/server/functions/validateName.ts new file mode 100644 index 000000000000..adb8c605fdac --- /dev/null +++ b/app/lib/server/functions/validateName.ts @@ -0,0 +1,14 @@ +import { settings } from '../../../settings/server'; + +export const validateName = function(name: string): boolean { + const blockedNames = settings.get('Accounts_SystemBlockedUsernameList'); + if (!blockedNames || typeof blockedNames !== 'string') { + return true; + } + + if (blockedNames.split(',').includes(name.toLowerCase())) { + return false; + } + + return true; +}; diff --git a/app/lib/server/methods/deleteUserOwnAccount.js b/app/lib/server/methods/deleteUserOwnAccount.js index 4a655856ec57..1ff7494a8751 100644 --- a/app/lib/server/methods/deleteUserOwnAccount.js +++ b/app/lib/server/methods/deleteUserOwnAccount.js @@ -1,6 +1,7 @@ import { Meteor } from 'meteor/meteor'; import { check } from 'meteor/check'; import { Accounts } from 'meteor/accounts-base'; +import { SHA256 } from 'meteor/sha'; import s from 'underscore.string'; import { settings } from '../../../settings'; @@ -34,7 +35,7 @@ Meteor.methods({ if (result.error) { throw new Meteor.Error('error-invalid-password', 'Invalid password', { method: 'deleteUserOwnAccount' }); } - } else if (user.username !== s.trim(password)) { + } else if (SHA256(user.username) !== s.trim(password)) { throw new Meteor.Error('error-invalid-username', 'Invalid username', { method: 'deleteUserOwnAccount' }); } diff --git a/app/lib/server/startup/settings.js b/app/lib/server/startup/settings.js index f7d67f0c484a..2d71653cc283 100644 --- a/app/lib/server/startup/settings.js +++ b/app/lib/server/startup/settings.js @@ -164,6 +164,10 @@ settings.addGroup('Accounts', function() { this.add('Accounts_BlockedUsernameList', '', { type: 'string', }); + this.add('Accounts_SystemBlockedUsernameList', 'admin,administrator,system,user', { + type: 'string', + hidden: true, + }); this.add('Accounts_UseDefaultBlockedDomainsList', true, { type: 'boolean', }); @@ -616,6 +620,26 @@ settings.addGroup('Accounts', function() { enableQuery, }); }); + + this.section('Password_History', function() { + this.add('Accounts_Password_History_Enabled', false, { + type: 'boolean', + i18nLabel: 'Enable_Password_History', + i18nDescription: 'Enable_Password_History_Description', + }); + + const enableQuery = { + _id: 'Accounts_Password_History_Enabled', + value: true, + }; + + this.add('Accounts_Password_History_Amount', 5, { + type: 'int', + enableQuery, + i18nLabel: 'Password_History_Amount', + i18nDescription: 'Password_History_Amount_Description', + }); + }); }); settings.addGroup('OAuth', function() { diff --git a/app/lib/tests/server.tests.js b/app/lib/tests/server.tests.js index 2c6957e2ffe8..cc6a4de04b1a 100644 --- a/app/lib/tests/server.tests.js +++ b/app/lib/tests/server.tests.js @@ -1,6 +1,8 @@ /* eslint-env mocha */ import 'babel-polyfill'; import assert from 'assert'; + +import { expect } from 'chai'; import './server.mocks.js'; import PasswordPolicyClass from '../server/lib/PasswordPolicyClass'; @@ -9,31 +11,31 @@ describe('PasswordPolicyClass', () => { describe('Default options', () => { const passwordPolice = new PasswordPolicyClass(); it('should be disabled', () => { - assert.equal(passwordPolice.enabled, false); + expect(passwordPolice.enabled).to.be.equal(false); }); it('should have minLength = -1', () => { - assert.equal(passwordPolice.minLength, -1); + expect(passwordPolice.minLength).to.be.equal(-1); }); it('should have maxLength = -1', () => { - assert.equal(passwordPolice.maxLength, -1); + expect(passwordPolice.maxLength).to.be.equal(-1); }); it('should have forbidRepeatingCharacters = false', () => { - assert.equal(passwordPolice.forbidRepeatingCharacters, false); + expect(passwordPolice.forbidRepeatingCharacters).to.be.false; }); it('should have forbidRepeatingCharactersCount = 3', () => { - assert.equal(passwordPolice.forbidRepeatingCharactersCount, 3); + expect(passwordPolice.forbidRepeatingCharactersCount).to.be.equal(3); }); it('should have mustContainAtLeastOneLowercase = false', () => { - assert.equal(passwordPolice.mustContainAtLeastOneLowercase, false); + expect(passwordPolice.mustContainAtLeastOneLowercase).to.be.false; }); it('should have mustContainAtLeastOneUppercase = false', () => { - assert.equal(passwordPolice.mustContainAtLeastOneUppercase, false); + expect(passwordPolice.mustContainAtLeastOneUppercase).to.be.false; }); it('should have mustContainAtLeastOneNumber = false', () => { - assert.equal(passwordPolice.mustContainAtLeastOneNumber, false); + expect(passwordPolice.mustContainAtLeastOneNumber).to.be.false; }); it('should have mustContainAtLeastOneSpecialCharacter = false', () => { - assert.equal(passwordPolice.mustContainAtLeastOneSpecialCharacter, false); + expect(passwordPolice.mustContainAtLeastOneSpecialCharacter).to.be.false; }); describe('Password tests with default options', () => { @@ -54,13 +56,12 @@ describe('PasswordPolicyClass', () => { enabled: true, throwError: false, }); - - assert.equal(passwordPolice.validate(), false); - assert.equal(passwordPolice.validate(1), false); - assert.equal(passwordPolice.validate(true), false); - assert.equal(passwordPolice.validate(new Date()), false); - assert.equal(passwordPolice.validate(new Function()), false); - assert.equal(passwordPolice.validate(''), false); + expect(passwordPolice.validate()).to.be.false; + expect(passwordPolice.validate(1)).to.be.false; + expect(passwordPolice.validate(true)).to.be.false; + expect(passwordPolice.validate(new Date())).to.be.false; + expect(passwordPolice.validate(new Function())).to.be.false; + expect(passwordPolice.validate('')).to.be.false; }); it('should restrict by minLength', () => { @@ -70,10 +71,10 @@ describe('PasswordPolicyClass', () => { throwError: false, }); - assert.equal(passwordPolice.validate('1'), false); - assert.equal(passwordPolice.validate('1234'), false); - assert.equal(passwordPolice.validate('12345'), true); - assert.equal(passwordPolice.validate(' '), false); + expect(passwordPolice.validate('1')).to.be.false; + expect(passwordPolice.validate('1234')).to.be.false; + expect(passwordPolice.validate('12345')).to.be.true; + expect(passwordPolice.validate(' ')).to.be.false; }); it('should restrict by maxLength', () => { @@ -83,10 +84,10 @@ describe('PasswordPolicyClass', () => { throwError: false, }); - assert.equal(passwordPolice.validate('1'), true); - assert.equal(passwordPolice.validate('12345'), true); - assert.equal(passwordPolice.validate('123456'), false); - assert.equal(passwordPolice.validate(' '), false); + expect(passwordPolice.validate('1')).to.be.true; + expect(passwordPolice.validate('12345')).to.be.true; + expect(passwordPolice.validate('123456')).to.be.false; + expect(passwordPolice.validate(' ')).to.be.false; }); it('should allow repeated characters', () => { @@ -96,11 +97,11 @@ describe('PasswordPolicyClass', () => { throwError: false, }); - assert.equal(passwordPolice.validate('1'), true); - assert.equal(passwordPolice.validate('12345'), true); - assert.equal(passwordPolice.validate('123456'), true); - assert.equal(passwordPolice.validate(' '), false); - assert.equal(passwordPolice.validate('11111111111111'), true); + expect(passwordPolice.validate('1')).to.be.true; + expect(passwordPolice.validate('12345')).to.be.true; + expect(passwordPolice.validate('123456')).to.be.true; + expect(passwordPolice.validate(' ')).to.be.false; + expect(passwordPolice.validate('11111111111111')).to.be.true; }); it('should restrict repeated characters', () => { @@ -111,12 +112,12 @@ describe('PasswordPolicyClass', () => { throwError: false, }); - assert.equal(passwordPolice.validate('1'), true); - assert.equal(passwordPolice.validate('11'), true); - assert.equal(passwordPolice.validate('111'), true); - assert.equal(passwordPolice.validate('1111'), false); - assert.equal(passwordPolice.validate(' '), false); - assert.equal(passwordPolice.validate('123456'), true); + expect(passwordPolice.validate('1')).to.be.true; + expect(passwordPolice.validate('11')).to.be.true; + expect(passwordPolice.validate('111')).to.be.true; + expect(passwordPolice.validate('1111')).to.be.false; + expect(passwordPolice.validate(' ')).to.be.false; + expect(passwordPolice.validate('123456')).to.be.true; }); it('should restrict repeated characters customized', () => { @@ -127,14 +128,14 @@ describe('PasswordPolicyClass', () => { throwError: false, }); - assert.equal(passwordPolice.validate('1'), true); - assert.equal(passwordPolice.validate('11'), true); - assert.equal(passwordPolice.validate('111'), true); - assert.equal(passwordPolice.validate('1111'), true); - assert.equal(passwordPolice.validate('11111'), true); - assert.equal(passwordPolice.validate('111111'), false); - assert.equal(passwordPolice.validate(' '), false); - assert.equal(passwordPolice.validate('123456'), true); + expect(passwordPolice.validate('1')).to.be.true; + expect(passwordPolice.validate('11')).to.be.true; + expect(passwordPolice.validate('111')).to.be.true; + expect(passwordPolice.validate('1111')).to.be.true; + expect(passwordPolice.validate('11111')).to.be.true; + expect(passwordPolice.validate('111111')).to.be.false; + expect(passwordPolice.validate(' ')).to.be.false; + expect(passwordPolice.validate('123456')).to.be.true; }); it('should contain one lowercase', () => { @@ -144,13 +145,13 @@ describe('PasswordPolicyClass', () => { throwError: false, }); - assert.equal(passwordPolice.validate('a'), true); - assert.equal(passwordPolice.validate('aa'), true); - assert.equal(passwordPolice.validate('A'), false); - assert.equal(passwordPolice.validate(' '), false); - assert.equal(passwordPolice.validate('123456'), false); - assert.equal(passwordPolice.validate('AAAAA'), false); - assert.equal(passwordPolice.validate('AAAaAAA'), true); + expect(passwordPolice.validate('a')).to.be.true; + expect(passwordPolice.validate('aa')).to.be.true; + expect(passwordPolice.validate('A')).to.be.false; + expect(passwordPolice.validate(' ')).to.be.false; + expect(passwordPolice.validate('123456')).to.be.false; + expect(passwordPolice.validate('AAAAA')).to.be.false; + expect(passwordPolice.validate('AAAaAAA')).to.be.true; }); it('should contain one uppercase', () => { @@ -160,48 +161,108 @@ describe('PasswordPolicyClass', () => { throwError: false, }); - assert.equal(passwordPolice.validate('a'), false); - assert.equal(passwordPolice.validate('aa'), false); - assert.equal(passwordPolice.validate('A'), true); - assert.equal(passwordPolice.validate(' '), false); - assert.equal(passwordPolice.validate('123456'), false); - assert.equal(passwordPolice.validate('AAAAA'), true); - assert.equal(passwordPolice.validate('AAAaAAA'), true); + expect(passwordPolice.validate('a')).to.be.false; + expect(passwordPolice.validate('aa')).to.be.false; + expect(passwordPolice.validate('A')).to.be.true; + expect(passwordPolice.validate(' ')).to.be.false; + expect(passwordPolice.validate('123456')).to.be.false; + expect(passwordPolice.validate('AAAAA')).to.be.true; + expect(passwordPolice.validate('AAAaAAA')).to.be.true; }); - it('should contain one uppercase', () => { + it('should contain one number', () => { const passwordPolice = new PasswordPolicyClass({ enabled: true, mustContainAtLeastOneNumber: true, throwError: false, }); - assert.equal(passwordPolice.validate('a'), false); - assert.equal(passwordPolice.validate('aa'), false); - assert.equal(passwordPolice.validate('A'), false); - assert.equal(passwordPolice.validate(' '), false); - assert.equal(passwordPolice.validate('123456'), true); - assert.equal(passwordPolice.validate('AAAAA'), false); - assert.equal(passwordPolice.validate('AAAaAAA'), false); - assert.equal(passwordPolice.validate('AAAa1AAA'), true); + expect(passwordPolice.validate('a')).to.be.false; + expect(passwordPolice.validate('aa')).to.be.false; + expect(passwordPolice.validate('A')).to.be.false; + expect(passwordPolice.validate(' ')).to.be.false; + expect(passwordPolice.validate('123456')).to.be.true; + expect(passwordPolice.validate('AAAAA')).to.be.false; + expect(passwordPolice.validate('AAAaAAA')).to.be.false; + expect(passwordPolice.validate('AAAa1AAA')).to.be.true; }); - it('should contain one uppercase', () => { + it('should contain one special character', () => { const passwordPolice = new PasswordPolicyClass({ enabled: true, mustContainAtLeastOneSpecialCharacter: true, throwError: false, }); - assert.equal(passwordPolice.validate('a'), false); - assert.equal(passwordPolice.validate('aa'), false); - assert.equal(passwordPolice.validate('A'), false); - assert.equal(passwordPolice.validate(' '), false); - assert.equal(passwordPolice.validate('123456'), false); - assert.equal(passwordPolice.validate('AAAAA'), false); - assert.equal(passwordPolice.validate('AAAaAAA'), false); - assert.equal(passwordPolice.validate('AAAa1AAA'), false); - assert.equal(passwordPolice.validate('AAAa@AAA'), true); + expect(passwordPolice.validate('a')).to.be.false; + expect(passwordPolice.validate('aa')).to.be.false; + expect(passwordPolice.validate('A')).to.be.false; + expect(passwordPolice.validate(' ')).to.be.false; + expect(passwordPolice.validate('123456')).to.be.false; + expect(passwordPolice.validate('AAAAA')).to.be.false; + expect(passwordPolice.validate('AAAaAAA')).to.be.false; + expect(passwordPolice.validate('AAAa1AAA')).to.be.false; + expect(passwordPolice.validate('AAAa@AAA')).to.be.true; + }); + }); + + describe('Password generator', () => { + it('should return a random password', () => { + const passwordPolice = new PasswordPolicyClass({ + enabled: true, + throwError: false, + }); + + expect(passwordPolice.generatePassword()).to.not.be.undefined; + }); + }); + + describe('Password Policy', () => { + it('should return a correct password policy', () => { + const passwordPolice = new PasswordPolicyClass({ + enabled: true, + throwError: false, + minLength: 10, + maxLength: 20, + forbidRepeatingCharacters: true, + forbidRepeatingCharactersCount: 4, + mustContainAtLeastOneLowercase: true, + mustContainAtLeastOneUppercase: true, + mustContainAtLeastOneNumber: true, + mustContainAtLeastOneSpecialCharacter: true, + }); + + const policy = passwordPolice.getPasswordPolicy(); + + expect(policy).to.not.be.undefined; + expect(policy.enabled).to.be.true; + expect(policy.policy.length).to.be.equal(8); + expect(policy.policy[0][0]).to.be.equal('get-password-policy-minLength'); + expect(policy.policy[0][1].minLength).to.be.equal(10); + }); + + it('should return correct values if policy is disabled', () => { + const passwordPolice = new PasswordPolicyClass({ + enabled: false, + }); + + const policy = passwordPolice.getPasswordPolicy(); + + expect(policy.enabled).to.be.false; + expect(policy.policy.length).to.be.equal(0); + }); + + it('should return correct values if policy is enabled but no specifiers exists', () => { + const passwordPolice = new PasswordPolicyClass({ + enabled: true, + }); + + const policy = passwordPolice.getPasswordPolicy(); + + expect(policy.enabled).to.be.true; + // even when no policy is specified, forbidRepeatingCharactersCount is still configured + // since its default value is 3 + expect(policy.policy.length).to.be.equal(1); }); }); }); diff --git a/app/livechat/client/tabBar.ts b/app/livechat/client/tabBar.ts index 0e1240d56a50..1ef8df95426c 100644 --- a/app/livechat/client/tabBar.ts +++ b/app/livechat/client/tabBar.ts @@ -7,7 +7,7 @@ addAction('room-info', { id: 'room-info', title: 'Room_Info', icon: 'info-circled', - template: lazy(() => import('../../../client/omnichannel/chats/contextualBar')), + template: lazy(() => import('../../../client/views/omnichannel/directory/chats/contextualBar/ChatsContextualBar')), order: 0, }); diff --git a/app/livechat/imports/server/rest/sms.js b/app/livechat/imports/server/rest/sms.js index 4bea9e09bec0..855ba255ddde 100644 --- a/app/livechat/imports/server/rest/sms.js +++ b/app/livechat/imports/server/rest/sms.js @@ -101,22 +101,28 @@ API.v1.addRoute('livechat/sms-incoming/:service', { const uploadedFile = getUploadFile(details, smsUrl); file = { _id: uploadedFile._id, name: uploadedFile.name, type: uploadedFile.type }; - const url = FileUpload.getPath(`${ file._id }/${ encodeURI(file.name) }`); + const fileUrl = FileUpload.getPath(`${ file._id }/${ encodeURI(file.name) }`); const attachment = { - message_link: url, + title: file.name, + type: 'file', + description: file.description, + title_link: fileUrl, }; - switch (contentType.substr(0, contentType.indexOf('/'))) { - case 'image': - attachment.image_url = url; - break; - case 'video': - attachment.video_url = url; - break; - case 'audio': - attachment.audio_url = url; - break; + if (/^image\/.+/.test(file.type)) { + attachment.image_url = fileUrl; + attachment.image_type = file.type; + attachment.image_size = file.size; + attachment.image_dimensions = file.identify != null ? file.identify.size : undefined; + } else if (/^audio\/.+/.test(file.type)) { + attachment.audio_url = fileUrl; + attachment.audio_type = file.type; + attachment.audio_size = file.size; + } else if (/^video\/.+/.test(file.type)) { + attachment.video_url = fileUrl; + attachment.video_type = file.type; + attachment.video_size = file.size; } attachments = [attachment]; diff --git a/app/livechat/lib/messageTypes.js b/app/livechat/lib/messageTypes.js index d31179643ca5..bde52192cbe9 100644 --- a/app/livechat/lib/messageTypes.js +++ b/app/livechat/lib/messageTypes.js @@ -80,3 +80,25 @@ MessageTypes.registerType({ system: true, message: 'New_videocall_request', }); + +MessageTypes.registerType({ + id: 'omnichannel_placed_chat_on_hold', + system: true, + message: 'Omnichannel_placed_chat_on_hold', + data(message) { + return { + comment: message.comment, + }; + }, +}); + +MessageTypes.registerType({ + id: 'omnichannel_on_hold_chat_resumed', + system: true, + message: 'Omnichannel_on_hold_chat_resumed', + data(message) { + return { + comment: message.comment, + }; + }, +}); diff --git a/app/livechat/server/lib/Contacts.js b/app/livechat/server/lib/Contacts.js index 7e0f94e10d7c..5e8b70aaf69b 100644 --- a/app/livechat/server/lib/Contacts.js +++ b/app/livechat/server/lib/Contacts.js @@ -4,6 +4,10 @@ import s from 'underscore.string'; import { LivechatVisitors, LivechatCustomField, + LivechatRooms, + Rooms, + LivechatInquiry, + Subscriptions, } from '../../../models'; @@ -60,6 +64,15 @@ export const Contacts = { LivechatVisitors.updateById(contactId, updateUser); + const rooms = LivechatRooms.findByVisitorId(contactId).fetch(); + + rooms?.length && rooms.forEach((room) => { + const { _id: rid } = room; + Rooms.setFnameById(rid, name) + && LivechatInquiry.setNameByRoomId(rid, name) + && Subscriptions.updateDisplayNameByRoomId(rid, name); + }); + return contactId; }, }; diff --git a/app/livechat/server/lib/Livechat.js b/app/livechat/server/lib/Livechat.js index 634d1af13eec..2bf6c70e6f2f 100644 --- a/app/livechat/server/lib/Livechat.js +++ b/app/livechat/server/lib/Livechat.js @@ -394,6 +394,7 @@ export const Livechat = { LivechatRooms.closeByRoomId(rid, closeData); LivechatInquiry.removeByRoomId(rid); + Subscriptions.removeByRoomId(rid); const message = { t: 'livechat-close', @@ -407,9 +408,7 @@ export const Livechat = { sendMessage(user || visitor, message, room); - if (servedBy) { - Subscriptions.removeByRoomIdAndUserId(rid, servedBy._id); - } + Messages.createCommandWithRoomIdAndUser('promptTranscript', rid, closeData.closedBy); Meteor.defer(() => { @@ -682,6 +681,10 @@ export const Livechat = { throw new Meteor.Error('error-returning-inquiry', 'Error returning inquiry to the queue', { method: 'livechat:returnRoomAsInquiry' }); } + Meteor.defer(() => { + callbacks.run('livechat:afterReturnRoomAsInquiry', { room }); + }); + return true; }, diff --git a/app/livechat/server/methods/loadHistory.js b/app/livechat/server/methods/loadHistory.js index cad161d5c161..910413a2f631 100644 --- a/app/livechat/server/methods/loadHistory.js +++ b/app/livechat/server/methods/loadHistory.js @@ -15,7 +15,7 @@ Meteor.methods({ throw new Meteor.Error('invalid-visitor', 'Invalid Visitor', { method: 'livechat:loadHistory' }); } - const room = LivechatRooms.findOneByIdAndVisitorToken(rid, token, { fields: { _id: 1 } }); + const room = LivechatRooms.findOneByIdAndVisitorToken(rid, token, { _id: 1 }); if (!room) { throw new Meteor.Error('invalid-room', 'Invalid Room', { method: 'livechat:loadHistory' }); } diff --git a/app/livestream/client/tabBar.tsx b/app/livestream/client/tabBar.tsx index f3d391985780..33e1fbb997de 100644 --- a/app/livestream/client/tabBar.tsx +++ b/app/livestream/client/tabBar.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from 'react'; +import React, { ReactNode, useMemo } from 'react'; import { Option, Badge } from '@rocket.chat/fuselage'; import { useSetting } from '../../../client/contexts/SettingsContext'; @@ -19,10 +19,10 @@ addAction('livestream', ({ room }) => { icon: 'podcast', template: 'liveStreamTab', order: isLive ? -1 : 15, - renderAction: (props): React.ReactNode => + renderAction: (props): ReactNode => {isLive ? ! : null} , - renderOption: ({ label: { title, icon }, ...props }: any): React.ReactNode => , } : null), [enabled, isLive, t]); diff --git a/app/mail-messages/client/index.js b/app/mail-messages/client/index.js index 09634cf5172b..8ad6156d106a 100644 --- a/app/mail-messages/client/index.js +++ b/app/mail-messages/client/index.js @@ -1,2 +1 @@ import './startup'; -import './router'; diff --git a/app/mail-messages/client/router.js b/app/mail-messages/client/router.js deleted file mode 100644 index 73bb53fac23e..000000000000 --- a/app/mail-messages/client/router.js +++ /dev/null @@ -1,12 +0,0 @@ -import { Meteor } from 'meteor/meteor'; -import { FlowRouter } from 'meteor/kadira:flow-router'; -import { BlazeLayout } from 'meteor/kadira:blaze-layout'; - -FlowRouter.route('/mailer/unsubscribe/:_id/:createdAt', { - name: 'mailer-unsubscribe', - async action(params) { - await import('./views'); - Meteor.call('Mailer:unsubscribe', params._id, params.createdAt); - return BlazeLayout.render('mailerUnsubscribe'); - }, -}); diff --git a/app/mail-messages/client/startup.js b/app/mail-messages/client/startup.js index 4de68df26552..6df18c09aa2b 100644 --- a/app/mail-messages/client/startup.js +++ b/app/mail-messages/client/startup.js @@ -5,7 +5,5 @@ registerAdminSidebarItem({ href: 'admin-mailer', i18nLabel: 'Mailer', icon: 'mail', - permissionGranted() { - return hasAllPermission('access-mailer'); - }, + permissionGranted: () => hasAllPermission('access-mailer'), }); diff --git a/app/mail-messages/client/views/index.js b/app/mail-messages/client/views/index.js deleted file mode 100644 index 81a1668b492f..000000000000 --- a/app/mail-messages/client/views/index.js +++ /dev/null @@ -1,2 +0,0 @@ -import './mailerUnsubscribe.html'; -import './mailerUnsubscribe'; diff --git a/app/mail-messages/client/views/mailerUnsubscribe.html b/app/mail-messages/client/views/mailerUnsubscribe.html deleted file mode 100644 index 637e0a596214..000000000000 --- a/app/mail-messages/client/views/mailerUnsubscribe.html +++ /dev/null @@ -1,14 +0,0 @@ - diff --git a/app/mail-messages/client/views/mailerUnsubscribe.js b/app/mail-messages/client/views/mailerUnsubscribe.js deleted file mode 100644 index b17122c8542d..000000000000 --- a/app/mail-messages/client/views/mailerUnsubscribe.js +++ /dev/null @@ -1,5 +0,0 @@ -import { Template } from 'meteor/templating'; - -Template.mailerUnsubscribe.onRendered(function() { - return $('#initial-page-loading').remove(); -}); diff --git a/app/markdown/lib/parser/original/markdown.js b/app/markdown/lib/parser/original/markdown.js index 637037ac57d6..f716ef4e7496 100644 --- a/app/markdown/lib/parser/original/markdown.js +++ b/app/markdown/lib/parser/original/markdown.js @@ -33,6 +33,51 @@ const validateUrl = (url, message) => { } }; +const endsWithWhitespace = (text) => text.substring(text.length - 1).match(/\s/); + +const getParseableMarkersCount = (start, end) => { + const usableMarkers = start.length > 1 ? 2 : 1; + return end.length - usableMarkers >= 0 ? usableMarkers : 1; +}; + +const getTextWrapper = (marker, tagName) => (textPrepend, wrappedText, textAppend) => + `${ textPrepend }${ marker }<${ tagName }>${ wrappedText }${ marker }${ textAppend }`; + +const getRegexReplacer = (replaceFunction, getRegex) => (marker, tagName) => { + const wrapper = getTextWrapper(marker, tagName); + return (msg) => msg.replace( + getRegex(marker), + (...args) => replaceFunction(wrapper, ...args), + ); +}; + +const getParserWithCustomMarker = getRegexReplacer( + (wrapper, match, p1, p2, p3) => { + if (endsWithWhitespace(p2)) { + return match; + } + const finalMarkerCount = getParseableMarkersCount(p1, p3); + return wrapper(p1.substring(finalMarkerCount), p2, p3.substring(finalMarkerCount)); + }, + (marker) => new RegExp(`(\\${ marker }+(?!\\s))([^\\${ marker }\\r\\n]+)(\\${ marker }+)`, 'gm'), +); + +const parseBold = getParserWithCustomMarker('*', 'strong'); + +const parseStrike = getParserWithCustomMarker('~', 'strike'); + +const parseItalic = getRegexReplacer( + (wrapper, match, p1, p2, p3, p4, p5) => { + if (p1 || p5 || endsWithWhitespace(p3)) { + return match; + } + + const finalMarkerCount = getParseableMarkersCount(p2, p4); + return wrapper(p2.substring(finalMarkerCount), p3, p4.substring(finalMarkerCount)); + }, + () => new RegExp('([^\\r\\n\\s~*_]){0,1}(\\_+(?!\\s))([^\\_\\r\\n]+)(\\_+)([^\\r\\n\\s]){0,1}', 'gm'), +)('_', 'em'); + const parseNotEscaped = (message, { supportSchemesForLink, headers, @@ -60,13 +105,13 @@ const parseNotEscaped = (message, { } // Support *text* to make bold - msg = msg.replace(/(|>|[ >_~`])\*{1,2}([^\*\r\n]+)\*{1,2}([<_~`]|\B|\b|$)/gm, '$1*$2*$3'); + msg = parseBold(msg); // Support _text_ to make italics - msg = msg.replace(/(^|>|[ >*~`])\_{1,2}([^\_\r\n]+)\_{1,2}([<*~`]|\B|\b|$)/gm, '$1_$2_$3'); + msg = parseItalic(msg); - // Support ~text~ to strike through text - msg = msg.replace(/(^|>|[ >_*`])\~{1,2}([^~\r\n]+)\~{1,2}([<_*`]|\B|\b|$)/gm, '$1~$2~$3'); + // // Support ~text~ to strike through text + msg = parseStrike(msg); // Support for block quote // >>> diff --git a/app/markdown/tests/client.tests.js b/app/markdown/tests/client.tests.js index fd92206771d8..d755693fdf8a 100644 --- a/app/markdown/tests/client.tests.js +++ b/app/markdown/tests/client.tests.js @@ -19,10 +19,21 @@ const inlinecodeWrapper = (text) => wrapper(` {{# with checkboxData }} - {{> Checkbox }} +
+ {{> Checkbox . }} +
{{/with}} diff --git a/app/threads/client/flextab/thread.js b/app/threads/client/flextab/thread.js index 134af66c45b6..96e5092e6e86 100644 --- a/app/threads/client/flextab/thread.js +++ b/app/threads/client/flextab/thread.js @@ -3,7 +3,6 @@ import { Meteor } from 'meteor/meteor'; import { Mongo } from 'meteor/mongo'; import { Template } from 'meteor/templating'; import { Session } from 'meteor/session'; -import { HTML } from 'meteor/htmljs'; import { ReactiveDict } from 'meteor/reactive-dict'; import { Tracker } from 'meteor/tracker'; import { FlowRouter } from 'meteor/kadira:flow-router'; @@ -14,7 +13,6 @@ import { messageContext } from '../../../ui-utils/client/lib/messageContext'; import { upsertMessageBulk } from '../../../ui-utils/client/lib/RoomHistoryManager'; import { Messages } from '../../../models'; import { fileUpload } from '../../../ui/client/lib/fileUpload'; -import { createTemplateForComponent } from '../../../../client/reactAdapters'; import { dropzoneEvents, dropzoneHelpers } from '../../../ui/client/views/app/room'; import './thread.html'; import { getUserPreference } from '../../../utils'; @@ -23,21 +21,8 @@ import { callbacks } from '../../../callbacks/client'; import './messageBoxFollow'; import { getCommonRoomEvents } from '../../../ui/client/views/app/lib/getCommonRoomEvents'; -createTemplateForComponent('Checkbox', async () => { - const { CheckBox } = await import('@rocket.chat/fuselage'); - return { default: CheckBox }; -}, { - // eslint-disable-next-line new-cap - renderContainerView: () => HTML.DIV({ class: 'rcx-checkbox', style: 'display: flex;' }), -}); - const sort = { ts: 1 }; -createTemplateForComponent('ThreadComponent', () => import('../components/ThreadComponent'), { - // eslint-disable-next-line new-cap - renderContainerView: () => HTML.DIV({ class: 'contextual-bar', style: 'display: flex; height: 100%;' }), -}); - Template.thread.events({ ...dropzoneEvents, ...getCommonRoomEvents(), diff --git a/app/threads/client/flextab/threadlist.tsx b/app/threads/client/flextab/threadlist.tsx index 75650d3dd6ca..2041d2777d80 100644 --- a/app/threads/client/flextab/threadlist.tsx +++ b/app/threads/client/flextab/threadlist.tsx @@ -35,6 +35,6 @@ addAction('thread', (options) => { { unread > 0 && {unread} }
; }, - order: 4, + order: 2, } : null), [threadsEnabled, room.tunread?.length, room.tunreadUser?.length, room.tunreadGroup?.length]); }); diff --git a/app/token-login/client/login_token_client.js b/app/token-login/client/login_token_client.js index 4809a81f241d..d5e05c32bed3 100644 --- a/app/token-login/client/login_token_client.js +++ b/app/token-login/client/login_token_client.js @@ -1,7 +1,8 @@ import { Meteor } from 'meteor/meteor'; import { Accounts } from 'meteor/accounts-base'; import { FlowRouter } from 'meteor/kadira:flow-router'; -import { BlazeLayout } from 'meteor/kadira:blaze-layout'; + +import { appLayout } from '../../../client/lib/appLayout'; Meteor.loginWithLoginToken = function(token) { Accounts.callLoginMethod({ @@ -19,7 +20,7 @@ Meteor.loginWithLoginToken = function(token) { FlowRouter.route('/login-token/:token', { name: 'tokenLogin', action() { - BlazeLayout.render('loginLayout'); + appLayout.render('loginLayout'); Meteor.loginWithLoginToken(this.getParam('token')); }, }); diff --git a/app/ui-login/client/login/layout.js b/app/ui-login/client/login/layout.js index e836872201ce..8da8197db217 100644 --- a/app/ui-login/client/login/layout.js +++ b/app/ui-login/client/login/layout.js @@ -2,10 +2,6 @@ import { Template } from 'meteor/templating'; import { settings } from '../../../settings'; -Template.loginLayout.onRendered(function() { - $('#initial-page-loading').remove(); -}); - Template.loginLayout.helpers({ backgroundUrl() { const asset = settings.get('Assets_background'); diff --git a/app/ui-login/client/routes.js b/app/ui-login/client/routes.js index 95b6a7ca7c4c..19e601a9d6a5 100644 --- a/app/ui-login/client/routes.js +++ b/app/ui-login/client/routes.js @@ -1,9 +1,10 @@ import { FlowRouter } from 'meteor/kadira:flow-router'; -import { BlazeLayout } from 'meteor/kadira:blaze-layout'; + +import { appLayout } from '../../../client/lib/appLayout'; FlowRouter.route('/reset-password/:token', { name: 'resetPassword', action() { - BlazeLayout.render('loginLayout', { center: 'resetPassword' }); + appLayout.render('loginLayout', { center: 'resetPassword' }); }, }); diff --git a/app/ui-master/client/body.html b/app/ui-master/client/body.html new file mode 100644 index 000000000000..641116de7ce0 --- /dev/null +++ b/app/ui-master/client/body.html @@ -0,0 +1 @@ + diff --git a/app/ui-master/client/body.js b/app/ui-master/client/body.js new file mode 100644 index 000000000000..bbc29c31fb80 --- /dev/null +++ b/app/ui-master/client/body.js @@ -0,0 +1,122 @@ +import Clipboard from 'clipboard'; +import s from 'underscore.string'; +import { Meteor } from 'meteor/meteor'; +import { Match } from 'meteor/check'; +import { Session } from 'meteor/session'; +import { Template } from 'meteor/templating'; + +import { t } from '../../utils/client'; +import { chatMessages } from '../../ui'; +import { Layout, modal, popover, fireGlobalEvent, RoomManager } from '../../ui-utils'; +import { settings } from '../../settings'; +import { ChatSubscription } from '../../models'; + +import './body.html'; + +Template.body.onRendered(function() { + new Clipboard('.clipboard'); + + $(document.body).on('keydown', function(e) { + const unread = Session.get('unread'); + if (e.keyCode === 27 && (e.shiftKey === true || e.ctrlKey === true) && (unread != null) && unread !== '') { + e.preventDefault(); + e.stopPropagation(); + modal.open({ + title: t('Clear_all_unreads_question'), + type: 'warning', + confirmButtonText: t('Yes_clear_all'), + showCancelButton: true, + cancelButtonText: t('Cancel'), + confirmButtonColor: '#DD6B55', + }, function() { + const subscriptions = ChatSubscription.find({ + open: true, + }, { + fields: { + unread: 1, + alert: 1, + rid: 1, + t: 1, + name: 1, + ls: 1, + }, + }); + + subscriptions.forEach((subscription) => { + if (subscription.alert || subscription.unread > 0) { + Meteor.call('readMessages', subscription.rid); + } + }); + }); + } + }); + + $(document.body).on('keydown', function(e) { + const { target } = e; + if (e.ctrlKey === true || e.metaKey === true) { + popover.close(); + return; + } + if (!((e.keyCode > 45 && e.keyCode < 91) || e.keyCode === 8)) { + return; + } + + if (/input|textarea|select/i.test(target.tagName)) { + return; + } + if (target.id === 'pswp') { + return; + } + + popover.close(); + + if (document.querySelector('.rc-modal-wrapper dialog[open]')) { + return; + } + + const inputMessage = chatMessages[RoomManager.openedRoom] && chatMessages[RoomManager.openedRoom].input; + if (!inputMessage) { + return; + } + inputMessage.focus(); + }); + + const handleMessageLinkClick = (event) => { + const link = event.currentTarget; + if (link.origin === s.rtrim(Meteor.absoluteUrl(), '/') && /msg=([a-zA-Z0-9]+)/.test(link.search)) { + fireGlobalEvent('click-message-link', { link: link.pathname + link.search }); + } + }; + + this.autorun(() => { + if (Layout.isEmbedded()) { + $(document.body).on('click', 'a', handleMessageLinkClick); + } else { + $(document.body).off('click', 'a', handleMessageLinkClick); + } + }); + + this.autorun(function(c) { + const w = window; + const d = document; + const script = 'script'; + const l = 'dataLayer'; + const i = settings.get('GoogleTagManager_id'); + if (Match.test(i, String) && i.trim() !== '') { + c.stop(); + return (function(w, d, s, l, i) { + w[l] = w[l] || []; + w[l].push({ + 'gtm.start': new Date().getTime(), + event: 'gtm.js', + }); + const f = d.getElementsByTagName(s)[0]; + const j = d.createElement(s); + const dl = l !== 'dataLayer' ? `&l=${ l }` : ''; + j.async = true; + j.src = `//www.googletagmanager.com/gtm.js?id=${ i }${ dl }`; + return f.parentNode.insertBefore(j, f); + }(w, d, script, l, i)); + } + }); +}); diff --git a/app/ui-master/client/index.js b/app/ui-master/client/index.js index 67abd78bcfe5..3a38da4d2ba6 100644 --- a/app/ui-master/client/index.js +++ b/app/ui-master/client/index.js @@ -1,5 +1,5 @@ +import './body'; import './loading'; import './error.html'; import './logoLayout.html'; -import './main.html'; import './main'; diff --git a/app/ui-master/client/main.html b/app/ui-master/client/main.html index 7b657a62c76a..201a9ad78b63 100644 --- a/app/ui-master/client/main.html +++ b/app/ui-master/client/main.html @@ -1,55 +1,42 @@ - - -