feat: handle non-local files with grace #2135
Comments
SgtPooki
added
exp/intermediate
To determine if a file is local, I'm using the following commands: ipfs block stat --offline CID # ipfs.block.stat(cid, { offline: true })I’m retrieving all blocks for the current directory and checking if they are local or not by using: ipfs dag get CID # ipfs.dag.get(f.cid)I modified the
/**
* @param {IPFSService} ipfs
* @param {CID} cid
* @param {Object} options
* @param {string} options.path
* @param {boolean} [options.isRoot]
* @param {import('./utils').Sorting} options.sorting
*/
const dirStats = async (ipfs, cid, { path, isRoot, sorting }) => {
const entries = await all(ipfs.ls(cid)) || []
// Workarounds regression in IPFS HTTP Client that causes
// ls on empty dir to return list with that dir only.
// @see https://github.com/ipfs/js-ipfs/issues/3566
const res = (entries.length === 1 && entries[0].cid.toString() === cid.toString())
? []
: entries
const files = []
// precaution: there was a historical performance issue when too many dirs were present
let dirCount = 0
for (const f of res) {
const absPath = join(path, f.name)
let file = null
if (dirCount < 1000 && (f.type === 'directory' || f.type === 'dir')) {
dirCount += 1
file = fileFromStats({ ...await stat(ipfs, f.cid), path: absPath })
} else {
file = fileFromStats({ ...f, path: absPath })
}
// Check if file is local
const refs = await ipfs.dag.get(f.cid)
// Extracting all CIDs
const refsCids = refs.value.Links.map(link => link.Hash['/'])
// console.log(refsCids)
// console.log(`refs=${JSON.stringify(refs, null, 2)}`)
if (dirCount > 300) {
file.isLocal = undefined
} else
if (refsCids.length === 0) {
file.isLocal = true
} else {
file.isLocal = true
// for (const myCid of refsCids) {
const myCid = refsCids[0]
const cid = CID.decode(myCid)
console.log(cid)
try {
// console.log(`myCid=${myCid} cid=${cid.toString()}`)
// const res = await ipfs.block.stat(cid, { withLocal: true, offline: true })
const res = await ipfs.block.stat(cid, { offline: true })
console.log(`myCid=${cid}, res=${JSON.stringify(res, null, 2)}`)
// break
} catch (error) {
file.isLocal = false
console.error(`Error fetching stats for cid=${cid}:`, error)
// break
}
}
files.push(file)
}The UI looks like this:
PS: Try the Project Apollo Archives and XKCD archives |
I wanted to check whether the browser is offline or online in JavaScript by using the const [isOnline, setIsOnline] = useState(navigator.onLine)
const CHECK_INTERNET_STATUS = 60 * 1000
function isGoogleOnline () {
return new Promise((resolve) => {
const img = document.createElement('img')
img.onerror = () => resolve(false)
img.onload = () => resolve(true)
img.src = 'https://www.google.com/favicon.ico?_=' + new Date().getTime()
})
}
// Function to create a delay
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms))
// Function to check status and log the result
async function checkInternetStatus () {
let onlineStatus = navigator.onLine
if (onlineStatus) {
console.log('You seems online! Checking Google...')
// Wait for 3 seconds
await delay(3000)
const googleOnline = await isGoogleOnline()
console.log('Google is', googleOnline ? 'online' : 'offline')
onlineStatus = googleOnline
} else {
console.log('No internet connection.')
}
setIsOnline(onlineStatus)
}
// Use useEffect to run the check once on component mount and set up an interval
useEffect(() => {
// Event listeners for online and offline status
const handleOnline = () => checkInternetStatus()
const handleOffline = () => setIsOnline(false)
window.addEventListener('online', handleOnline)
window.addEventListener('offline', handleOffline)
// Check status immediately on page load
checkInternetStatus()
// Set up the interval to check status every 15 seconds
const intervalId = setInterval(checkInternetStatus, CHECK_INTERNET_STATUS)
// Clean up the interval and event listeners when the component unmounts
return () => {
clearInterval(intervalId)
window.removeEventListener('online', handleOnline)
window.removeEventListener('offline', handleOffline)
}
}, []) // Empty dependency array to run only once on mountIf we're offline I'll apply this class to shade/unclick the file view .greyed-out {
color: grey;
pointer-events: none; /* This will prevent clicking */
opacity: 0.5;
}Should I choose only to use Thoughts? |
|
hey @acul71, thanks for the ping. We shouldn't include a dependency on a service like google.. I think a simple check for a 'hello world' CID should be sufficient for a backup online check? such as a fetch for If we query the local kubo node it could resolve immediately (due to being an inline CID) even if we're not actually online, so i'm thinking maybe a light fetch request to trustless-gateway.link? 
const url = 'https://trustless-gateway.link/ipfs/bafkqae2xmvwgg33nmuqhi3zajfiemuzahiwss?format=raw';
async function headRequest(url) {
try {
const response = await fetch(url, {
method: 'HEAD',
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
// Log status code
console.log('Status Code:', response.status);
} catch (error) {
console.error('Error making HEAD request:', error);
}
}
headRequest(url); |
I'm planning to modify doFilesFetch that handles the procedure. /**
* Adds file under the `src` path to the given `root` path. On completion will
* trigger `doFilesFetch` to update the state.
* @param {string} root
* @param {string} src
* @param {string} name
*/
doFilesAddPath: (root, src, name = '', downloadLocally = false) => perform(ACTIONS.ADD_BY_PATH, async (ipfs, { store }) => {
ensureMFS(store)
const path = realMfsPath(src)
const cid = /** @type {string} */(path.split('/').pop())
if (!name) {
name = cid
}
const dst = realMfsPath(join(root, name))
const srcPath = src.startsWith('/') ? src : `/ipfs/${cid}`
try {
console.log(`Adding ipfs.files.cp(${srcPath}, ${dst}) downloadLocally=${downloadLocally}`)
const cid = await ipfs.files.cp(srcPath, dst)
console.log(`Adding ipfs.files.cp(${srcPath}, ${dst}) = ${cid}`)
// ipfs refs --recursive --help
if (downloadLocally === true) {
for await (const ref of ipfs.refs(srcPath, { recursive: true })) {
if (ref.err) {
console.error('refs: ', ref.err)
} else {
console.log('refs: ', ref)
// output: "QmHash"
}
}
}
return cid
} finally {
await store.doFilesFetch()
}
}), |
.......... lidel (from Slack) React 4:59 ############################## @lidel @SgtPooki time_X 43 files 0.152 seconds
time_Apollo_albums.txt 105 files 0.422 seconds
time_XKCD_Archive.txt ~1800 files 1.429 secondsMaybe what I can do is submit this solution in the PR and open an issue or feat request in kubo, to let this info (local or not) come back with ipfs.files.stat. /**
* @param {IPFSService} ipfs
* @param {CID} cid
* @param {Object} options
* @param {string} options.path
* @param {boolean} [options.isRoot]
* @param {import('./utils').Sorting} options.sorting
*/
const dirStats = async (ipfs, cid, { path, isRoot, sorting }) => {
const entries = await all(ipfs.ls(cid)) || []
// Workarounds regression in IPFS HTTP Client that causes
// ls on empty dir to return list with that dir only.
// @see https://github.com/ipfs/js-ipfs/issues/3566
const res = (entries.length === 1 && entries[0].cid.toString() === cid.toString())
? []
: entries
const files = []
// precaution: there was a historical performance issue when too many dirs were present
let dirCount = 0
// let fileCount = 0
let totDuration = 0
for (const f of res) {
const absPath = join(path, f.name)
let file = null
if (dirCount < 1000 && (f.type === 'directory' || f.type === 'dir')) {
dirCount += 1
file = fileFromStats({ ...await stat(ipfs, f.cid), path: absPath })
} else {
file = fileFromStats({ ...f, path: absPath })
}
// Start timing
const startTime = performance.now()
// If dirCount > 300 stop checking and mark isLocal as undefined
if (dirCount > 300) {
// if (fileCount > 300) {
file.isLocal = undefined
} else {
// fileCount += 1
// Check if file is local
// TODO: See if # ipfs ls --size=false --resolve-type=false CID is faster then ipfs.dag.get CID
const refs = await ipfs.dag.get(f.cid)
// Extracting all CIDs
// const refsCids = refs.value.Links.map(link => link.Hash['/'])
const firstRefCid = refs.value.Links.length > 0 ? refs.value.Links[0].Hash['/'] : ''
// console.log(refsCids)
// console.log(`refs=${JSON.stringify(refs, null, 2)}`)
// if (refsCids.length === 0) {
if (firstRefCid === '') {
file.isLocal = true
} else {
file.isLocal = true
// for (const myCid of refsCids) {
// const myCid = refsCids[0]
// const cid = CID.decode(myCid)
const cid = CID.decode(firstRefCid)
// console.log(cid) // debug lucas
try {
// console.log(`myCid=${myCid} cid=${cid.toString()}`)
// const res = await ipfs.block.stat(cid, { offline: true })
await ipfs.block.stat(cid, { offline: true })
// console.log(`myCid=${cid}, res=${JSON.stringify(res, null, 2)}`)
// break
} catch (error) {
file.isLocal = false
console.error(`Error fetching stats for cid=${cid}:`, error)
// break
}
}
}
// End timing and log the duration
const endTime = performance.now()
// const duration = (endTime - startTime) / 1000 // Convert to seconds
const duration = (endTime - startTime)
totDuration += duration
files.push(file)
}
console.log(`Time taken to check if file is local: ${totDuration / 1000} secs`)
let parent = null
if (!isRoot) {
const parentPath = dirname(path)
const parentInfo = infoFromPath(parentPath, false)
if (parentInfo && (parentInfo.isMfs || !parentInfo.isRoot)) {
const realPath = parentInfo.realPath
if (realPath && realPath.startsWith('/ipns')) {
parentInfo.realPath = await last(ipfs.name.resolve(parentInfo.realPath))
}
parent = fileFromStats({
...await stat(ipfs, parentInfo.realPath),
path: parentInfo.path,
name: '..',
isParent: true
})
}
}
return {
path,
fetched: Date.now(),
type: 'directory',
cid,
upper: parent,
content: sortFiles(files, sorting)
}
} |
|
What is the exact meaning of files, pins, blocks, local repo ? Right know It's like this: I assume that:
About the expandable thingy (▶) |
This task aims to resolve all of the issues in the
Related itemssection below. The tasklist clarifies how we plan to accomplish this.Tasks
Some details about a method forward for this at ipfs/ipfs-desktop#2492 (comment)
Related items
The text was updated successfully, but these errors were encountered: