feat: 优化日志读取 (#5485)

zhengkunwang223 · web-flow · commit 345dbda400fd · 2024-06-17T14:24:50.000Z
Refs #3690
diff --git a/backend/app/dto/request/file.go b/backend/app/dto/request/file.go
@@ -126,6 +126,7 @@ type FileReadByLineReq struct {
 	Type     string `json:"type" validate:"required"`
 	ID       uint   `json:"ID"`
 	Name     string `json:"name"`
+	Latest   bool   `json:"latest"`
 }
 
 type FileExistReq struct {
diff --git a/backend/app/dto/response/file.go b/backend/app/dto/response/file.go
@@ -37,6 +37,7 @@ type FileLineContent struct {
 	Content string `json:"content"`
 	End     bool   `json:"end"`
 	Path    string `json:"path"`
+	Total   int    `json:"total"`
 }
 
 type FileExist struct {
diff --git a/backend/app/service/file.go b/backend/app/service/file.go
@@ -410,14 +410,15 @@ func (f *FileService) ReadLogByLine(req request.FileReadByLineReq) (*response.Fi
 		logFilePath = path.Join(global.CONF.System.TmpDir, fmt.Sprintf("docker_logs/%s", req.Name))
 	}
 
-	lines, isEndOfFile, err := files.ReadFileByLine(logFilePath, req.Page, req.PageSize)
+	lines, isEndOfFile, total, err := files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)
 	if err != nil {
 		return nil, err
 	}
 	res := &response.FileLineContent{
 		Content: strings.Join(lines, "\n"),
 		End:     isEndOfFile,
 		Path:    logFilePath,
+		Total:   total,
 	}
 	return res, nil
 }
diff --git a/backend/app/service/website.go b/backend/app/service/website.go
@@ -1008,7 +1008,7 @@ func (w WebsiteService) OpWebsiteLog(req request.WebsiteLogReq) (*response.Websi
 			}
 		}
 		filePath := path.Join(sitePath, "log", req.LogType)
-		lines, end, err := files.ReadFileByLine(filePath, req.Page, req.PageSize)
+		lines, end, _, err := files.ReadFileByLine(filePath, req.Page, req.PageSize, false)
 		if err != nil {
 			return nil, err
 		}
diff --git a/backend/utils/files/utils.go b/backend/utils/files/utils.go
@@ -66,19 +66,44 @@ func IsHidden(path string) bool {
 	return path[0] == dotCharacter
 }
 
-func ReadFileByLine(filename string, page, pageSize int) ([]string, bool, error) {
+func countLines(path string) (int, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return 0, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	lineCount := 0
+	for scanner.Scan() {
+		lineCount++
+	}
+	if err := scanner.Err(); err != nil {
+		return 0, err
+	}
+	return lineCount, nil
+}
+
+func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []string, isEndOfFile bool, total int, err error) {
 	if !NewFileOp().Stat(filename) {
-		return nil, true, nil
+		return
 	}
 	file, err := os.Open(filename)
 	if err != nil {
-		return nil, false, err
+		return
 	}
 	defer file.Close()
 
+	totalLines, err := countLines(filename)
+	if err != nil {
+		return
+	}
+	total = (totalLines + pageSize - 1) / pageSize
 	reader := bufio.NewReaderSize(file, 8192)
 
-	var lines []string
+	if latest {
+		page = total
+	}
 	currentLine := 0
 	startLine := (page - 1) * pageSize
 	endLine := startLine + pageSize
@@ -97,9 +122,8 @@ func ReadFileByLine(filename string, page, pageSize int) ([]string, bool, error)
 		}
 	}
 
-	isEndOfFile := currentLine < endLine
-
-	return lines, isEndOfFile, nil
+	isEndOfFile = currentLine < endLine
+	return
 }
 
 func GetParentMode(path string) (os.FileMode, error) {
diff --git a/frontend/package.json b/frontend/package.json
@@ -1,7 +1,7 @@
 {
     "name": "1Panel-Frontend",
     "private": true,
-    "version": "1.7",
+    "version": "1.10",
     "description": "1Panel 前端",
     "scripts": {
         "dev": "vite",
@@ -25,13 +25,15 @@
         "@codemirror/legacy-modes": "^6.4.0",
         "@codemirror/theme-one-dark": "^6.1.2",
         "@element-plus/icons-vue": "^1.1.4",
+        "@highlightjs/vue-plugin": "^2.1.0",
         "@vueuse/core": "^8.9.4",
         "@xterm/addon-fit": "^0.10.0",
         "@xterm/xterm": "^5.5.0",
         "axios": "^1.7.2",
         "echarts": "^5.5.0",
         "element-plus": "^2.7.5",
         "fit2cloud-ui-plus": "^1.1.4",
+        "highlight.js": "^11.9.0",
         "js-base64": "^3.7.7",
         "md-editor-v3": "^2.11.3",
         "monaco-editor": "^0.34.1",
diff --git a/frontend/src/components/log-file/index.vue b/frontend/src/components/log-file/index.vue
@@ -12,35 +12,23 @@
             </span>
         </div>
         <div class="mt-2.5">
-            <Codemirror
-                ref="logContainer"
-                :style="styleObject"
-                :autofocus="true"
-                :placeholder="$t('website.noLog')"
-                :indent-with-tab="true"
-                :tabSize="4"
-                :lineWrapping="true"
-                :matchBrackets="true"
-                theme="cobalt"
-                :styleActiveLine="true"
-                :extensions="extensions"
-                v-model="content"
-                :disabled="true"
-                @ready="handleReady"
-            />
+            <highlightjs
+                ref="editorRef"
+                class="editor-main"
+                language="JavaScript"
+                :autodetect="false"
+                :code="content"
+            ></highlightjs>
         </div>
     </div>
 </template>
 <script lang="ts" setup>
-import { Codemirror } from 'vue-codemirror';
-import { javascript } from '@codemirror/lang-javascript';
-import { oneDark } from '@codemirror/theme-one-dark';
-import { computed, nextTick, onMounted, onUnmounted, reactive, ref, shallowRef } from 'vue';
+import { nextTick, onMounted, onUnmounted, reactive, ref } from 'vue';
 import { downloadFile } from '@/utils/util';
 import { ReadByLine } from '@/api/modules/files';
 import { watch } from 'vue';
 
-const extensions = [javascript(), oneDark];
+const editorRef = ref();
 
 interface LogProps {
     id?: number;
@@ -61,7 +49,7 @@ const props = defineProps({
     },
     style: {
         type: String,
-        default: 'height: calc(100vh - 200px); width: 100%; min-height: 400px',
+        default: 'height: calc(100vh - 200px); width: 100%; min-height: 400px; overflow: auto;',
     },
     defaultButton: {
         type: Boolean,
@@ -84,29 +72,23 @@ const data = ref({
 
 let timer: NodeJS.Timer | null = null;
 const tailLog = ref(false);
-const view = shallowRef();
 const content = ref('');
 const end = ref(false);
 const lastContent = ref('');
-const logContainer = ref();
 const scrollerElement = ref<HTMLElement | null>(null);
+const minPage = ref(1);
+const maxPage = ref(1);
 
 const readReq = reactive({
     id: 0,
     type: '',
     name: '',
-    page: 0,
-    pageSize: 2000,
+    page: 1,
+    pageSize: 500,
+    latest: false,
 });
 const emit = defineEmits(['update:loading', 'update:hasContent', 'update:isReading']);
 
-const handleReady = (payload) => {
-    view.value = payload.view;
-    const editorContainer = payload.container;
-    const editorElement = editorContainer.querySelector('.cm-editor');
-    scrollerElement.value = editorElement.querySelector('.cm-scroller') as HTMLElement;
-};
-
 const loading = ref(props.loading);
 
 watch(
@@ -121,25 +103,6 @@ const changeLoading = () => {
     emit('update:loading', loading.value);
 };
 
-const styleObject = computed(() => {
-    const styles = {};
-    let style = 'height: calc(100vh - 200px); width: 100%; min-height: 400px';
-    if (props.style != null && props.style != '') {
-        style = props.style;
-    }
-    style.split(';').forEach((styleRule) => {
-        const [property, value] = styleRule.split(':');
-        if (property && value) {
-            const formattedProperty = property
-                .trim()
-                .replace(/([a-z])([A-Z])/g, '$1-$2')
-                .toLowerCase();
-            styles[formattedProperty] = value.trim();
-        }
-    });
-    return styles;
-});
-
 const stopSignals = [
     'docker-compose up failed!',
     'docker-compose up successful!',
@@ -151,18 +114,16 @@ const stopSignals = [
     'image push successful!',
 ];
 
-const getContent = () => {
+const getContent = (pre: boolean) => {
     emit('update:isReading', true);
-    if (!end.value) {
-        readReq.page += 1;
-    }
     readReq.id = props.config.id;
     readReq.type = props.config.type;
     readReq.name = props.config.name;
     ReadByLine(readReq).then((res) => {
         if (!end.value && res.data.end) {
             lastContent.value = content.value;
         }
+
         res.data.content = res.data.content.replace(/\\u(\w{4})/g, function (match, grp) {
             return String.fromCharCode(parseInt(grp, 16));
         });
@@ -175,28 +136,38 @@ const getContent = () => {
                 if (lastContent.value == '') {
                     content.value = res.data.content;
                 } else {
-                    content.value = lastContent.value + '\n' + res.data.content;
+                    content.value = pre
+                        ? res.data.content + '\n' + lastContent.value
+                        : lastContent.value + '\n' + res.data.content;
                 }
             } else {
                 if (content.value == '') {
                     content.value = res.data.content;
                 } else {
-                    content.value = content.value + '\n' + res.data.content;
+                    content.value = pre
+                        ? res.data.content + '\n' + content.value
+                        : content.value + '\n' + res.data.content;
                 }
             }
         }
         end.value = res.data.end;
         emit('update:hasContent', content.value !== '');
         nextTick(() => {
-            const state = view.value.state;
-            view.value.dispatch({
-                selection: { anchor: state.doc.length, head: state.doc.length },
-            });
-            view.value.focus();
-            const firstLine = view.value.state.doc.line(view.value.state.doc.lines);
-            const { top } = view.value.lineBlockAt(firstLine.from);
-            scrollerElement.value.scrollTo({ top, behavior: 'instant' });
+            if (pre) {
+                if (scrollerElement.value.scrollHeight > 2000) {
+                    scrollerElement.value.scrollTop = 2000;
+                }
+            } else {
+                scrollerElement.value.scrollTop = scrollerElement.value.scrollHeight;
+            }
         });
+
+        if (readReq.latest) {
+            readReq.page = res.data.total;
+            readReq.latest = false;
+            maxPage.value = res.data.total;
+            minPage.value = res.data.total;
+        }
     });
 };
 
@@ -206,7 +177,7 @@ const changeTail = (fromOutSide: boolean) => {
     }
     if (tailLog.value) {
         timer = setInterval(() => {
-            getContent();
+            getContent(false);
         }, 1000 * 2);
     } else {
         onCloseLog();
@@ -230,6 +201,10 @@ function isScrolledToBottom(element: HTMLElement): boolean {
     return element.scrollTop + element.clientHeight + 1 >= element.scrollHeight;
 }
 
+function isScrolledToTop(element: HTMLElement): boolean {
+    return element.scrollTop === 0;
+}
+
 const init = () => {
     if (props.config.tail) {
         tailLog.value = props.config.tail;
@@ -239,30 +214,56 @@ const init = () => {
     if (tailLog.value) {
         changeTail(false);
     }
-    getContent();
+    readReq.latest = true;
+    getContent(false);
+
+    nextTick(() => {});
+};
+
+const clearLog = (): void => {
+    content.value = '';
+};
 
+const initCodemirror = () => {
     nextTick(() => {
-        if (scrollerElement.value) {
+        if (editorRef.value) {
+            scrollerElement.value = editorRef.value.$el as HTMLElement;
             scrollerElement.value.addEventListener('scroll', function () {
                 if (isScrolledToBottom(scrollerElement.value)) {
-                    getContent();
+                    readReq.page = maxPage.value;
+                    getContent(false);
+                }
+                if (isScrolledToTop(scrollerElement.value)) {
+                    readReq.page = minPage.value - 1;
+                    if (readReq.page < 1) {
+                        return;
+                    }
+                    minPage.value = readReq.page;
+                    getContent(true);
                 }
             });
+            let hljsDom = scrollerElement.value.querySelector('.hljs') as HTMLElement;
+            hljsDom.style['min-height'] = '300px';
         }
     });
 };
 
-const clearLog = (): void => {
-    content.value = '';
-};
-
 onUnmounted(() => {
     onCloseLog();
 });
 
 onMounted(() => {
+    initCodemirror();
     init();
 });
 
 defineExpose({ changeTail, onDownload, clearLog });
 </script>
+<style lang="scss" scoped>
+.editor-main {
+    height: calc(100vh - 480px);
+    width: 100%;
+    min-height: 400px;
+    overflow: auto;
+}
+</style>
diff --git a/frontend/src/main.ts b/frontend/src/main.ts
diff --git a/frontend/src/views/website/website/config/index.vue b/frontend/src/views/website/website/config/index.vue

Original file line number	Diff line number	Diff line change
`@@ -126,6 +126,7 @@ type FileReadByLineReq struct {`
`126`	`126`	Type string `json:"type" validate:"required"`
`127`	`127`	ID uint `json:"ID"`
`128`	`128`	Name string `json:"name"`
	`129`	+ Latest bool `json:"latest"`
`129`	`130`	`}`
`130`	`131`
`131`	`132`	`type FileExistReq struct {`
Original file line number	Diff line number	Diff line change
`@@ -37,6 +37,7 @@ type FileLineContent struct {`
`37`	`37`	Content string `json:"content"`
`38`	`38`	End bool `json:"end"`
`39`	`39`	Path string `json:"path"`
	`40`	+ Total int `json:"total"`
`40`	`41`	`}`
`41`	`42`
`42`	`43`	`type FileExist struct {`
Original file line number	Diff line number	Diff line change
`@@ -410,14 +410,15 @@ func (f FileService) ReadLogByLine(req request.FileReadByLineReq) (response.Fi`
`410`	`410`	`logFilePath = path.Join(global.CONF.System.TmpDir, fmt.Sprintf("docker_logs/%s", req.Name))`
`411`	`411`	`}`
`412`	`412`
`413`		`- lines, isEndOfFile, err := files.ReadFileByLine(logFilePath, req.Page, req.PageSize)`
	`413`	`+ lines, isEndOfFile, total, err := files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)`
`414`	`414`	`if err != nil {`
`415`	`415`	`return nil, err`
`416`	`416`	`}`
`417`	`417`	`res := &response.FileLineContent{`
`418`	`418`	`Content: strings.Join(lines, "\n"),`
`419`	`419`	`End: isEndOfFile,`
`420`	`420`	`Path: logFilePath,`
	`421`	`+ Total: total,`
`421`	`422`	`}`
`422`	`423`	`return res, nil`
`423`	`424`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1008,7 +1008,7 @@ func (w WebsiteService) OpWebsiteLog(req request.WebsiteLogReq) (*response.Websi`
`1008`	`1008`	`}`
`1009`	`1009`	`}`
`1010`	`1010`	`filePath := path.Join(sitePath, "log", req.LogType)`
`1011`		`- lines, end, err := files.ReadFileByLine(filePath, req.Page, req.PageSize)`
	`1011`	`+ lines, end, _, err := files.ReadFileByLine(filePath, req.Page, req.PageSize, false)`
`1012`	`1012`	`if err != nil {`
`1013`	`1013`	`return nil, err`
`1014`	`1014`	`}`