Skip to content

Commit

Permalink
refactor: simplify code by listing files recursively
Browse files Browse the repository at this point in the history
  • Loading branch information
@erezrokah
erezrokah committed Dec 6, 2020
1 parent d915617 commit cd217d3
Show file tree
Hide file tree
Showing 12 changed files with 126 additions and 186 deletions.
63 changes: 19 additions & 44 deletions packages/netlify-cms-backend-github/src/API.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,13 @@ export interface Config {
cmsLabelPrefix: string;
}

export enum TreeFileType {
TREE = 'tree',
BLOB = 'blob',
}

interface TreeFile {
type: 'blob' | 'tree';
type: TreeFileType;
sha: string;
path: string;
raw?: string;
Expand Down Expand Up @@ -383,7 +388,9 @@ export default class API {
this.request(`${this.repoURL}/git/trees`, {
method: 'POST',
body: JSON.stringify({
tree: [{ path: 'README.md', mode: '100644', type: 'blob', sha: item.sha }],
tree: [
{ path: 'README.md', mode: '100644', type: TreeFileType.BLOB, sha: item.sha },
],
}),
}),
)
Expand Down Expand Up @@ -661,8 +668,8 @@ export default class API {

async listFiles(
path: string,
{ repoURL = this.repoURL, branch = this.branch, depth = 1 } = {},
): Promise<{ type: string; id: string; name: string; path: string; size: number }[]> {
{ repoURL = this.repoURL, branch = this.branch, depth = 1, types = [TreeFileType.BLOB] } = {},
): Promise<{ type: TreeFileType; id: string; name: string; path: string; size: number }[]> {
const folder = trim(path, '/');
try {
const result: Octokit.GitGetTreeResponse = await this.request(
Expand All @@ -676,9 +683,12 @@ export default class API {
return (
result.tree
// filter only files and up to the required depth
.filter(file => file.type === 'blob' && file.path.split('/').length <= depth)
.filter(
file =>
types.includes(file.type as TreeFileType) && file.path.split('/').length <= depth,
)
.map(file => ({
type: file.type,
type: file.type as TreeFileType,
id: file.sha,
name: basename(file.path),
path: `${folder}/${file.path}`,
Expand All @@ -695,41 +705,6 @@ export default class API {
}
}

async listDirs(
path: string,
{ repoURL = this.repoURL, branch = this.branch, depth = 1 } = {},
): Promise<{ type: string; id: string; name: string; path: string; }[]> {
const folder = trim(path, '/');
try {
const result: Octokit.GitGetTreeResponse = await this.request(
`${repoURL}/git/trees/${branch}:${folder}`,
{
// GitHub API supports recursive=1 for getting the entire recursive tree
// or omitting it to get the non-recursive tree
params: depth > 1 ? { recursive: 1 } : {},
},
);
return (
result.tree
// filter only files and up to the required depth
.filter(file => file.type === 'tree' && file.path.split('/').length <= depth)
.map(file => ({
type: file.type,
id: file.sha,
name: basename(file.path),
path: `${folder}/${file.path}`,
}))
);
} catch (err) {
if (err && err.status === 404) {
console.log('This 404 was expected and handled appropriately.');
return [];
} else {
throw err;
}
}
}

filterOpenAuthoringBranches = async (branch: string) => {
try {
const pullRequest = await this.getBranchPullRequest(branch);
Expand Down Expand Up @@ -1404,7 +1379,7 @@ export default class API {
const entry = {
path: trimStart(file.path, '/'),
mode: '100644',
type: 'blob',
type: TreeFileType.BLOB,
sha: file.sha,
} as TreeEntry;

Expand All @@ -1426,14 +1401,14 @@ export default class API {
tree.push({
path: file.path,
mode: '100644',
type: 'blob',
type: TreeFileType.BLOB,
sha: null,
});
// create in new path
tree.push({
path: file.path.replace(sourceDir, destDir),
mode: '100644',
type: 'blob',
type: TreeFileType.BLOB,
sha: file.path === from ? sha : file.id,
});
}
Expand Down
50 changes: 31 additions & 19 deletions packages/netlify-cms-backend-github/src/GraphQLAPI.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,14 @@ import {
} from 'netlify-cms-lib-util';
import { trim, trimStart } from 'lodash';
import introspectionQueryResultData from './fragmentTypes';
import API, { Config, BlobArgs, API_NAME, PullRequestState, MOCK_PULL_REQUEST } from './API';
import API, {
Config,
BlobArgs,
API_NAME,
PullRequestState,
MOCK_PULL_REQUEST,
TreeFileType,
} from './API';
import * as queries from './queries';
import * as mutations from './mutations';
import { GraphQLError } from 'graphql';
Expand All @@ -35,7 +42,7 @@ interface TreeEntry {
object?: {
entries: TreeEntry[];
};
type: 'blob' | 'tree';
type: TreeFileType;
name: string;
sha: string;
blob?: {
Expand All @@ -47,7 +54,7 @@ interface TreeFile {
path: string;
id: string;
size: number;
type: string;
type: TreeFileType;
name: string;
}

Expand Down Expand Up @@ -337,30 +344,35 @@ export default class GraphQLAPI extends API {
}
}

getAllFiles(entries: TreeEntry[], path: string) {
getAllFiles(entries: TreeEntry[], path: string, types: TreeFileType[]) {
const allFiles: TreeFile[] = entries.reduce((acc, item) => {
if (item.type === 'tree') {
const itemToAdd = {
name: item.name,
type: item.type,
id: item.sha,
path: `${path}/${item.name}`,
size: item.blob ? item.blob.size : 0,
};
if (item.type === TreeFileType.BLOB) {
return [...acc, itemToAdd];
}

if (item.type === TreeFileType.TREE) {
const entries = item.object?.entries || [];
return [...acc, ...this.getAllFiles(entries, `${path}/${item.name}`)];
} else if (item.type === 'blob') {
return [
...acc,
{
name: item.name,
type: item.type,
id: item.sha,
path: `${path}/${item.name}`,
size: item.blob ? item.blob.size : 0,
},
];
const currentDir = types.includes(TreeFileType.TREE) ? [itemToAdd] : [];
const subFiles = this.getAllFiles(entries, `${path}/${item.name}`, types);
return [...acc, ...currentDir, ...subFiles];
}

return acc;
}, [] as TreeFile[]);
return allFiles;
}

async listFiles(path: string, { repoURL = this.repoURL, branch = this.branch, depth = 1 } = {}) {
async listFiles(
path: string,
{ repoURL = this.repoURL, branch = this.branch, depth = 1, types = [TreeFileType.BLOB] } = {},
) {
const { owner, name } = this.getOwnerAndNameFromRepoUrl(repoURL);
const folder = trim(path, '/');
const { data } = await this.query({
Expand All @@ -369,7 +381,7 @@ export default class GraphQLAPI extends API {
});

if (data.repository.object) {
const allFiles = this.getAllFiles(data.repository.object.entries, folder);
const allFiles = this.getAllFiles(data.repository.object.entries, folder, types);
return allFiles;
} else {
return [];
Expand Down
48 changes: 21 additions & 27 deletions packages/netlify-cms-backend-github/src/implementation.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import {
} from 'netlify-cms-lib-util';
import AuthenticationPage from './AuthenticationPage';
import { Octokit } from '@octokit/rest';
import API, { API_NAME } from './API';
import API, { API_NAME, TreeFileType } from './API';
import GraphQLAPI from './GraphQLAPI';

type GitHubUser = Octokit.UsersGetAuthenticatedResponse;
Expand Down Expand Up @@ -69,8 +69,7 @@ export default class GitHub implements Implementation {
useOpenAuthoring?: boolean;
branch: string;
apiRoot: string;
defaultMediaFolder: string;
currentMediaFolder: string;
mediaFolder: string;
previewContext: string;
token: string | null;
squashMerges: boolean;
Expand Down Expand Up @@ -116,8 +115,7 @@ export default class GitHub implements Implementation {
this.squashMerges = config.backend.squash_merges || false;
this.cmsLabelPrefix = config.backend.cms_label_prefix || '';
this.useGraphql = config.backend.use_graphql || false;
this.defaultMediaFolder = config.media_folder;
this.currentMediaFolder = config.media_folder;
this.mediaFolder = config.media_folder;
this.previewContext = config.backend.preview_context || '';
this.lock = asyncLock();
}
Expand Down Expand Up @@ -438,25 +436,24 @@ export default class GitHub implements Implementation {
.catch(() => ({ file: { path, id: null }, data: '' }));
}

updateMediaFolder(path: string) {
this.currentMediaFolder = path || this.currentMediaFolder;
return this.currentMediaFolder;
}

getDefaultMediaFolder() {
return this.defaultMediaFolder;
}

getMedia(currentMediaFolder = this.currentMediaFolder) {
return Promise.all([this.api!.listDirs(currentMediaFolder), this.api!.listFiles(currentMediaFolder)]).then(result => {
return result[0].map(({ id, name, path, type }) => {
return { id, name, displayURL: { id, path }, path, type };
}).concat(result[1].map(({ id, name, size, path, type }) => {
// load media using getMediaDisplayURL to avoid token expiration with GitHub raw content urls
// for private repositories
return { id, name, size, displayURL: { id, path }, path, type };
}));
})
async getMedia(mediaFolder = this.mediaFolder) {
const files = await this.api!.listFiles(mediaFolder, {
types: [TreeFileType.BLOB, TreeFileType.TREE],
depth: 100,
});
const withDisplayUrls = files.map(({ id, name, size, path, type }) => {
// load media using getMediaDisplayURL to avoid token expiration with GitHub raw content urls
// for private repositories
return {
id,
name,
size,
displayURL: { id, path },
path,
isDirectory: type === TreeFileType.TREE,
};
});
return withDisplayUrls;
}

async getMediaFile(path: string) {
Expand Down Expand Up @@ -497,12 +494,9 @@ export default class GitHub implements Implementation {
}

async persistMedia(mediaFile: AssetProxy, options: PersistOptions) {
console.log({...mediaFile});
try {
await this.api!.persistFiles([], [mediaFile], options);
const { sha, path, fileObj } = mediaFile as AssetProxy & { sha: string };
console.log(mediaFile);
console.log(path)
const displayURL = URL.createObjectURL(fileObj);
return {
id: sha,
Expand Down
22 changes: 4 additions & 18 deletions packages/netlify-cms-core/src/actions/mediaLibrary.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,6 @@ export const MEDIA_DELETE_FAILURE = 'MEDIA_DELETE_FAILURE';
export const MEDIA_DISPLAY_URL_REQUEST = 'MEDIA_DISPLAY_URL_REQUEST';
export const MEDIA_DISPLAY_URL_SUCCESS = 'MEDIA_DISPLAY_URL_SUCCESS';
export const MEDIA_DISPLAY_URL_FAILURE = 'MEDIA_DISPLAY_URL_FAILURE';
export const MEDIA_FOLDER_UPDATE = 'MEDIA_FOLDER_UPDATE';
export const DEFAULT_MEDIA_FOLDER = 'DEFAULT_MEDIA_FOLDER';

export function createMediaLibrary(instance: MediaLibraryInstance) {
const api = {
Expand Down Expand Up @@ -133,19 +131,9 @@ export function removeInsertedMedia(controlID: string) {
return { type: MEDIA_REMOVE_INSERTED, payload: { controlID } };
}

export function updateMediaFolder(selectedMediaFolder: string) {
return (dispatch: ThunkDispatch<State, {}, AnyAction>, getState: () => State) => {
const state = getState();
const backend = currentBackend(state.config);
const currentMediaFolder = backend.updateMediaFolder(selectedMediaFolder);
const defaultMediaFolder = backend.getDefaultMediaFolder();
dispatch({ type: MEDIA_FOLDER_UPDATE, payload: { currentMediaFolder, defaultMediaFolder } });
};
}

export function loadMedia(
opts: { delay?: number; query?: string; page?: number; privateUpload?: boolean } = {}
) {
export function loadMedia(
opts: { delay?: number; query?: string; page?: number; privateUpload?: boolean } = {},
) {
const { delay = 0, query = '', page = 1, privateUpload } = opts;
return async (dispatch: ThunkDispatch<State, {}, AnyAction>, getState: () => State) => {
const state = getState();
Expand All @@ -169,6 +157,7 @@ export function loadMedia(
}
}
dispatch(mediaLoading(page));

const loadFunction = () =>
backend
.getMedia()
Expand Down Expand Up @@ -273,7 +262,6 @@ export function persistMedia(file: File, opts: MediaOptions = {}) {
} else {
const entry = state.entryDraft.get('entry');
const collection = state.collections.get(entry?.get('collection'));
console.log(state);
const path = selectMediaFilePath(state.config, collection, entry, fileName, field);
assetProxy = createAssetProxy({
file,
Expand All @@ -299,8 +287,6 @@ export function persistMedia(file: File, opts: MediaOptions = {}) {
});
return dispatch(addDraftEntryMediaFile(mediaFile));
} else {
console.log('backend.persistMedia');
console.log(state.config);
mediaFile = await backend.persistMedia(state.config, assetProxy);
}

Expand Down
8 changes: 0 additions & 8 deletions packages/netlify-cms-core/src/backend.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -805,14 +805,6 @@ export class Backend {
return entryValue;
}

updateMediaFolder(path: string) {
return this.implementation.updateMediaFolder(path);
}

getDefaultMediaFolder() {
return this.implementation.getDefaultMediaFolder();
}

getMedia() {
return this.implementation.getMedia();
}
Expand Down
Loading

0 comments on commit cd217d3

Please sign in to comment.