Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: 优化日志读取 #5485

Merged
merged 1 commit into from
Jun 17, 2024
Merged

feat: 优化日志读取 #5485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions backend/app/dto/request/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ type FileReadByLineReq struct {
Type string `json:"type" validate:"required"`
ID uint `json:"ID"`
Name string `json:"name"`
Latest bool `json:"latest"`
}

type FileExistReq struct {
Expand Down
1 change: 1 addition & 0 deletions backend/app/dto/response/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type FileLineContent struct {
Content string `json:"content"`
End bool `json:"end"`
Path string `json:"path"`
Total int `json:"total"`
}

type FileExist struct {
Expand Down
3 changes: 2 additions & 1 deletion backend/app/service/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,14 +410,15 @@ func (f *FileService) ReadLogByLine(req request.FileReadByLineReq) (*response.Fi
logFilePath = path.Join(global.CONF.System.TmpDir, fmt.Sprintf("docker_logs/%s", req.Name))
}

lines, isEndOfFile, err := files.ReadFileByLine(logFilePath, req.Page, req.PageSize)
lines, isEndOfFile, total, err := files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)
if err != nil {
return nil, err
}
res := &response.FileLineContent{
Content: strings.Join(lines, "\n"),
End: isEndOfFile,
Path: logFilePath,
Total: total,
}
return res, nil
}
2 changes: 1 addition & 1 deletion backend/app/service/website.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1008,7 +1008,7 @@ func (w WebsiteService) OpWebsiteLog(req request.WebsiteLogReq) (*response.Websi
}
}
filePath := path.Join(sitePath, "log", req.LogType)
lines, end, err := files.ReadFileByLine(filePath, req.Page, req.PageSize)
lines, end, _, err := files.ReadFileByLine(filePath, req.Page, req.PageSize, false)
if err != nil {
return nil, err
}
Expand Down
38 changes: 31 additions & 7 deletions backend/utils/files/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,19 +66,44 @@ func IsHidden(path string) bool {
return path[0] == dotCharacter
}

func ReadFileByLine(filename string, page, pageSize int) ([]string, bool, error) {
func countLines(path string) (int, error) {
file, err := os.Open(path)
if err != nil {
return 0, err
}
defer file.Close()

scanner := bufio.NewScanner(file)
lineCount := 0
for scanner.Scan() {
lineCount++
}
if err := scanner.Err(); err != nil {
return 0, err
}
return lineCount, nil
}

func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []string, isEndOfFile bool, total int, err error) {
if !NewFileOp().Stat(filename) {
return nil, true, nil
return
}
file, err := os.Open(filename)
if err != nil {
return nil, false, err
return
}
defer file.Close()

totalLines, err := countLines(filename)
if err != nil {
return
}
total = (totalLines + pageSize - 1) / pageSize
reader := bufio.NewReaderSize(file, 8192)

var lines []string
if latest {
page = total
}
currentLine := 0
startLine := (page - 1) * pageSize
endLine := startLine + pageSize
Expand All @@ -97,9 +122,8 @@ func ReadFileByLine(filename string, page, pageSize int) ([]string, bool, error)
}
}

isEndOfFile := currentLine < endLine

return lines, isEndOfFile, nil
isEndOfFile = currentLine < endLine
return
}

func GetParentMode(path string) (os.FileMode, error) {
Expand Down
4 changes: 3 additions & 1 deletion frontend/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "1Panel-Frontend",
"private": true,
"version": "1.7",
"version": "1.10",
"description": "1Panel 前端",
"scripts": {
"dev": "vite",
Expand All @@ -25,13 +25,15 @@
"@codemirror/legacy-modes": "^6.4.0",
"@codemirror/theme-one-dark": "^6.1.2",
"@element-plus/icons-vue": "^1.1.4",
"@highlightjs/vue-plugin": "^2.1.0",
"@vueuse/core": "^8.9.4",
"@xterm/addon-fit": "^0.10.0",
"@xterm/xterm": "^5.5.0",
"axios": "^1.7.2",
"echarts": "^5.5.0",
"element-plus": "^2.7.5",
"fit2cloud-ui-plus": "^1.1.4",
"highlight.js": "^11.9.0",
"js-base64": "^3.7.7",
"md-editor-v3": "^2.11.3",
"monaco-editor": "^0.34.1",
Expand Down
149 changes: 75 additions & 74 deletions frontend/src/components/log-file/index.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,35 +12,23 @@
</span>
</div>
<div class="mt-2.5">
<Codemirror
ref="logContainer"
:style="styleObject"
:autofocus="true"
:placeholder="$t('website.noLog')"
:indent-with-tab="true"
:tabSize="4"
:lineWrapping="true"
:matchBrackets="true"
theme="cobalt"
:styleActiveLine="true"
:extensions="extensions"
v-model="content"
:disabled="true"
@ready="handleReady"
/>
<highlightjs
ref="editorRef"
class="editor-main"
language="JavaScript"
:autodetect="false"
:code="content"
></highlightjs>
</div>
</div>
</template>
<script lang="ts" setup>
import { Codemirror } from 'vue-codemirror';
import { javascript } from '@codemirror/lang-javascript';
import { oneDark } from '@codemirror/theme-one-dark';
import { computed, nextTick, onMounted, onUnmounted, reactive, ref, shallowRef } from 'vue';
import { nextTick, onMounted, onUnmounted, reactive, ref } from 'vue';
import { downloadFile } from '@/utils/util';
import { ReadByLine } from '@/api/modules/files';
import { watch } from 'vue';

const extensions = [javascript(), oneDark];
const editorRef = ref();

interface LogProps {
id?: number;
Expand All @@ -61,7 +49,7 @@ const props = defineProps({
},
style: {
type: String,
default: 'height: calc(100vh - 200px); width: 100%; min-height: 400px',
default: 'height: calc(100vh - 200px); width: 100%; min-height: 400px; overflow: auto;',
},
defaultButton: {
type: Boolean,
Expand All @@ -84,29 +72,23 @@ const data = ref({

let timer: NodeJS.Timer | null = null;
const tailLog = ref(false);
const view = shallowRef();
const content = ref('');
const end = ref(false);
const lastContent = ref('');
const logContainer = ref();
const scrollerElement = ref<HTMLElement | null>(null);
const minPage = ref(1);
const maxPage = ref(1);

const readReq = reactive({
id: 0,
type: '',
name: '',
page: 0,
pageSize: 2000,
page: 1,
pageSize: 500,
latest: false,
});
const emit = defineEmits(['update:loading', 'update:hasContent', 'update:isReading']);

const handleReady = (payload) => {
view.value = payload.view;
const editorContainer = payload.container;
const editorElement = editorContainer.querySelector('.cm-editor');
scrollerElement.value = editorElement.querySelector('.cm-scroller') as HTMLElement;
};

const loading = ref(props.loading);

watch(
Expand All @@ -121,25 +103,6 @@ const changeLoading = () => {
emit('update:loading', loading.value);
};

const styleObject = computed(() => {
const styles = {};
let style = 'height: calc(100vh - 200px); width: 100%; min-height: 400px';
if (props.style != null && props.style != '') {
style = props.style;
}
style.split(';').forEach((styleRule) => {
const [property, value] = styleRule.split(':');
if (property && value) {
const formattedProperty = property
.trim()
.replace(/([a-z])([A-Z])/g, '$1-$2')
.toLowerCase();
styles[formattedProperty] = value.trim();
}
});
return styles;
});

const stopSignals = [
'docker-compose up failed!',
'docker-compose up successful!',
Expand All @@ -151,18 +114,16 @@ const stopSignals = [
'image push successful!',
];

const getContent = () => {
const getContent = (pre: boolean) => {
emit('update:isReading', true);
if (!end.value) {
readReq.page += 1;
}
readReq.id = props.config.id;
readReq.type = props.config.type;
readReq.name = props.config.name;
ReadByLine(readReq).then((res) => {
if (!end.value && res.data.end) {
lastContent.value = content.value;
}

res.data.content = res.data.content.replace(/\\u(\w{4})/g, function (match, grp) {
return String.fromCharCode(parseInt(grp, 16));
});
Expand All @@ -175,28 +136,38 @@ const getContent = () => {
if (lastContent.value == '') {
content.value = res.data.content;
} else {
content.value = lastContent.value + '\n' + res.data.content;
content.value = pre
? res.data.content + '\n' + lastContent.value
: lastContent.value + '\n' + res.data.content;
}
} else {
if (content.value == '') {
content.value = res.data.content;
} else {
content.value = content.value + '\n' + res.data.content;
content.value = pre
? res.data.content + '\n' + content.value
: content.value + '\n' + res.data.content;
}
}
}
end.value = res.data.end;
emit('update:hasContent', content.value !== '');
nextTick(() => {
const state = view.value.state;
view.value.dispatch({
selection: { anchor: state.doc.length, head: state.doc.length },
});
view.value.focus();
const firstLine = view.value.state.doc.line(view.value.state.doc.lines);
const { top } = view.value.lineBlockAt(firstLine.from);
scrollerElement.value.scrollTo({ top, behavior: 'instant' });
if (pre) {
if (scrollerElement.value.scrollHeight > 2000) {
scrollerElement.value.scrollTop = 2000;
}
} else {
scrollerElement.value.scrollTop = scrollerElement.value.scrollHeight;
}
});

if (readReq.latest) {
readReq.page = res.data.total;
readReq.latest = false;
maxPage.value = res.data.total;
minPage.value = res.data.total;
}
});
};

Expand All @@ -206,7 +177,7 @@ const changeTail = (fromOutSide: boolean) => {
}
if (tailLog.value) {
timer = setInterval(() => {
getContent();
getContent(false);
}, 1000 * 2);
} else {
onCloseLog();
Expand All @@ -230,6 +201,10 @@ function isScrolledToBottom(element: HTMLElement): boolean {
return element.scrollTop + element.clientHeight + 1 >= element.scrollHeight;
}

function isScrolledToTop(element: HTMLElement): boolean {
return element.scrollTop === 0;
}

const init = () => {
if (props.config.tail) {
tailLog.value = props.config.tail;
Expand All @@ -239,30 +214,56 @@ const init = () => {
if (tailLog.value) {
changeTail(false);
}
getContent();
readReq.latest = true;
getContent(false);

nextTick(() => {});
};

const clearLog = (): void => {
content.value = '';
};

const initCodemirror = () => {
nextTick(() => {
if (scrollerElement.value) {
if (editorRef.value) {
scrollerElement.value = editorRef.value.$el as HTMLElement;
scrollerElement.value.addEventListener('scroll', function () {
if (isScrolledToBottom(scrollerElement.value)) {
getContent();
readReq.page = maxPage.value;
getContent(false);
}
if (isScrolledToTop(scrollerElement.value)) {
readReq.page = minPage.value - 1;
if (readReq.page < 1) {
return;
}
minPage.value = readReq.page;
getContent(true);
}
});
let hljsDom = scrollerElement.value.querySelector('.hljs') as HTMLElement;
hljsDom.style['min-height'] = '300px';
}
});
};

const clearLog = (): void => {
content.value = '';
};

onUnmounted(() => {
onCloseLog();
});

onMounted(() => {
initCodemirror();
init();
});

defineExpose({ changeTail, onDownload, clearLog });
</script>
<style lang="scss" scoped>
.editor-main {
height: calc(100vh - 480px);
width: 100%;
min-height: 400px;
overflow: auto;
}
</style>
Loading
Loading