From be1d46a424e1618dc5f678ce2d3cf984d4db732f Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Wed, 15 Jan 2025 18:17:45 +0530 Subject: [PATCH 01/12] Caching extended for imageSource --- packages/base/src/tools.ts | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index d2f847949..daf5d951c 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -457,6 +457,25 @@ export const loadGeoTIFFWithCache = async (sourceInfo: { }; }; +const getFromCache = async (key: string) => { + const cachedData = await getFromIndexedDB(key); + if (cachedData) { + return { + data: new Blob([cachedData.file]), + metadata: cachedData.metadata + }; + } + return null; +}; + +const saveToCache = async ( + key: string, + data: Blob, + metadata?: any +) => { + await saveToIndexedDB(key, data, metadata); +}; + /** * Generalized file reader for different source types. * @@ -473,7 +492,15 @@ export const loadFile = async (fileInfo: { if (filepath.startsWith('http://') || filepath.startsWith('https://')) { switch (type) { case 'ImageSource': { - return filepath; // Return the URL directly + const cached = await getFromCache(filepath); + if (cached) { + return URL.createObjectURL(cached.data); // Use Object URL directly + } + const response = await fetch(filepath); + const fileBlob = await response.blob(); + await saveToCache(filepath, fileBlob); + + return URL.createObjectURL(fileBlob); } case 'ShapefileSource': { From 5afc120469d284690c7b859e627b3f351011c48f Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Wed, 15 Jan 2025 18:26:46 +0530 Subject: [PATCH 02/12] Make caching work for shapefiles --- packages/base/src/tools.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index daf5d951c..1d3f728e3 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -504,12 +504,18 @@ export const loadFile = async (fileInfo: { } case 'ShapefileSource': { + const cached = await getFromCache(filepath); + if (cached) { + return cached.data.text(); + } + try { const response = await fetch( `/jupytergis_core/proxy?url=${filepath}` ); const arrayBuffer = await response.arrayBuffer(); const geojson = await shp(arrayBuffer); + await saveToCache(filepath, new Blob([JSON.stringify(geojson)])); return geojson; } catch (error) { console.error('Error loading remote shapefile:', error); From fcf7aec2446ed5d77d466e989b777fcc8bb3e890 Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Wed, 15 Jan 2025 18:34:01 +0530 Subject: [PATCH 03/12] Works for geojson too --- packages/base/src/tools.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index 1d3f728e3..1c4a87eef 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -524,6 +524,11 @@ export const loadFile = async (fileInfo: { } case 'GeoJSONSource': { + const cached = await getFromCache(filepath); + if (cached) { + return JSON.parse(await cached.data.text()); + } + try { const response = await fetch( `/jupytergis_core/proxy?url=${filepath}` @@ -532,6 +537,7 @@ export const loadFile = async (fileInfo: { throw new Error(`Failed to fetch GeoJSON from URL: ${filepath}`); } const geojson = await response.json(); + await saveToCache(filepath, new Blob([JSON.stringify(geojson)])); return geojson; } catch (error) { console.error('Error loading remote GeoJSON:', error); From 29c0216e3e54a69f2392e9a87dc2253884673f02 Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Wed, 15 Jan 2025 18:41:58 +0530 Subject: [PATCH 04/12] use abstracted methods for geotiff too --- packages/base/src/tools.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index 1c4a87eef..026e03839 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -429,10 +429,10 @@ export const loadGeoTIFFWithCache = async (sourceInfo: { return null; } - const cachedData = await getFromIndexedDB(sourceInfo.url); + const cachedData = await getFromCache(sourceInfo.url); if (cachedData) { return { - file: new Blob([cachedData.file]), + file: cachedData.data, metadata: cachedData.metadata, sourceUrl: sourceInfo.url }; @@ -448,7 +448,7 @@ export const loadGeoTIFFWithCache = async (sourceInfo: { const metadata = await Gdal.gdalinfo(tifDataset, ['-stats']); Gdal.close(tifDataset); - await saveToIndexedDB(sourceInfo.url, fileBlob, metadata); + await saveToCache(sourceInfo.url, fileBlob, metadata); return { file: fileBlob, @@ -506,7 +506,7 @@ export const loadFile = async (fileInfo: { case 'ShapefileSource': { const cached = await getFromCache(filepath); if (cached) { - return cached.data.text(); + return JSON.parse(await cached.data.text()); } try { From ac2943e8f703bd77bbfdf9be8a2c2713e10d2a53 Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Wed, 15 Jan 2025 18:48:22 +0530 Subject: [PATCH 05/12] lint --- packages/base/src/tools.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index 026e03839..bfa2a2354 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -468,11 +468,7 @@ const getFromCache = async (key: string) => { return null; }; -const saveToCache = async ( - key: string, - data: Blob, - metadata?: any -) => { +const saveToCache = async (key: string, data: Blob, metadata?: any) => { await saveToIndexedDB(key, data, metadata); }; From a7b0c6d99aad17d125b80b18fde575e9348c3b44 Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Tue, 21 Jan 2025 15:59:42 +0530 Subject: [PATCH 06/12] merge image path validation logic from #362 --- packages/base/src/tools.ts | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index 6fa1002e8..d48b823e9 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -492,11 +492,27 @@ export const loadFile = async (fileInfo: { if (cached) { return URL.createObjectURL(cached.data); // Use Object URL directly } - const response = await fetch(filepath); - const fileBlob = await response.blob(); - await saveToCache(filepath, fileBlob); - return URL.createObjectURL(fileBlob); + try { + const response = await fetch(filepath); + if (!response.ok) { + throw new Error(`Failed to fetch image from URL: ${filepath}`); + } + + const contentType = response.headers.get('Content-Type'); + if (!contentType || !contentType.startsWith('image/')) { + throw new Error(`Invalid image URL. Content-Type: ${contentType}`); + } + + const fileBlob = await response.blob(); + await validateImage(fileBlob); + await saveToCache(filepath, fileBlob); + + return URL.createObjectURL(fileBlob); + } catch (error) { + console.error('Error validating remote image:', error); + throw error; + } } case 'ShapefileSource': { From a3f62836dd0f79503b73d30adeb4af7985e1befd Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Tue, 21 Jan 2025 16:25:02 +0530 Subject: [PATCH 07/12] don't use caching logic on image --- packages/base/src/tools.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index d48b823e9..c96a2355e 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -488,11 +488,6 @@ export const loadFile = async (fileInfo: { if (filepath.startsWith('http://') || filepath.startsWith('https://')) { switch (type) { case 'ImageSource': { - const cached = await getFromCache(filepath); - if (cached) { - return URL.createObjectURL(cached.data); // Use Object URL directly - } - try { const response = await fetch(filepath); if (!response.ok) { @@ -504,11 +499,9 @@ export const loadFile = async (fileInfo: { throw new Error(`Invalid image URL. Content-Type: ${contentType}`); } - const fileBlob = await response.blob(); - await validateImage(fileBlob); - await saveToCache(filepath, fileBlob); - - return URL.createObjectURL(fileBlob); + // load the image to verify it's not corrupted + await validateImage(await response.blob()); + return filepath; } catch (error) { console.error('Error validating remote image:', error); throw error; From 9cb6c3667cb582b19833d974001ac9f8afc4463a Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Tue, 21 Jan 2025 17:11:55 +0530 Subject: [PATCH 08/12] lint --- packages/base/src/tools.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index c96a2355e..f3befe4ed 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -499,9 +499,9 @@ export const loadFile = async (fileInfo: { throw new Error(`Invalid image URL. Content-Type: ${contentType}`); } - // load the image to verify it's not corrupted - await validateImage(await response.blob()); - return filepath; + // load the image to verify it's not corrupted + await validateImage(await response.blob()); + return filepath; } catch (error) { console.error('Error validating remote image:', error); throw error; From 3576d526119dd05c01a1e3c0fb55cea7bc378368 Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Tue, 21 Jan 2025 22:45:19 +0530 Subject: [PATCH 09/12] refactor to straightup cache geojson --- packages/base/src/tools.ts | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index f3befe4ed..87a16a585 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -384,7 +384,10 @@ export const openDatabase = () => { */ export const saveToIndexedDB = async ( key: string, - file: Blob, + file: + | Blob + | shp.FeatureCollectionWithFilename + | shp.FeatureCollectionWithFilename[], metadata: any ) => { const db = await openDatabase(); @@ -460,15 +463,27 @@ export const loadGeoTIFFWithCache = async (sourceInfo: { const getFromCache = async (key: string) => { const cachedData = await getFromIndexedDB(key); if (cachedData) { + const data = + cachedData.file instanceof Blob + ? new Blob([cachedData.file]) + : cachedData.file; + return { - data: new Blob([cachedData.file]), + data, metadata: cachedData.metadata }; } return null; }; -const saveToCache = async (key: string, data: Blob, metadata?: any) => { +const saveToCache = async ( + key: string, + data: + | Blob + | shp.FeatureCollectionWithFilename + | shp.FeatureCollectionWithFilename[], + metadata?: any +) => { await saveToIndexedDB(key, data, metadata); }; @@ -511,7 +526,7 @@ export const loadFile = async (fileInfo: { case 'ShapefileSource': { const cached = await getFromCache(filepath); if (cached) { - return JSON.parse(await cached.data.text()); + return cached.data; } try { @@ -520,7 +535,7 @@ export const loadFile = async (fileInfo: { ); const arrayBuffer = await response.arrayBuffer(); const geojson = await shp(arrayBuffer); - await saveToCache(filepath, new Blob([JSON.stringify(geojson)])); + await saveToCache(filepath, geojson); return geojson; } catch (error) { console.error('Error loading remote shapefile:', error); @@ -531,7 +546,7 @@ export const loadFile = async (fileInfo: { case 'GeoJSONSource': { const cached = await getFromCache(filepath); if (cached) { - return JSON.parse(await cached.data.text()); + return cached.data; } try { @@ -542,7 +557,10 @@ export const loadFile = async (fileInfo: { throw new Error(`Failed to fetch GeoJSON from URL: ${filepath}`); } const geojson = await response.json(); - await saveToCache(filepath, new Blob([JSON.stringify(geojson)])); + await saveToCache( + filepath, + geojson as shp.FeatureCollectionWithFilename + ); return geojson; } catch (error) { console.error('Error loading remote GeoJSON:', error); From e4dfcf831eade5f8a04d7a010e1d05ec572611d7 Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Thu, 23 Jan 2025 15:43:15 +0530 Subject: [PATCH 10/12] Remove wrappers, cache any file type --- packages/base/src/tools.ts | 57 ++++++++++---------------------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index 87a16a585..4d9fb3f9e 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -384,11 +384,8 @@ export const openDatabase = () => { */ export const saveToIndexedDB = async ( key: string, - file: - | Blob - | shp.FeatureCollectionWithFilename - | shp.FeatureCollectionWithFilename[], - metadata: any + file: any, + metadata?: any | undefined ) => { const db = await openDatabase(); return new Promise((resolve, reject) => { @@ -409,7 +406,10 @@ export const saveToIndexedDB = async ( */ export const getFromIndexedDB = async (key: string) => { const db = await openDatabase(); - return new Promise((resolve, reject) => { + return new Promise<{ + file: any; + metadata?: any | undefined; + } | undefined>((resolve, reject) => { const transaction = db.transaction('files', 'readonly'); const store = transaction.objectStore('files'); const request = store.get(key); @@ -432,10 +432,10 @@ export const loadGeoTIFFWithCache = async (sourceInfo: { return null; } - const cachedData = await getFromCache(sourceInfo.url); + const cachedData = await getFromIndexedDB(sourceInfo.url); if (cachedData) { return { - file: cachedData.data, + file: cachedData.file, metadata: cachedData.metadata, sourceUrl: sourceInfo.url }; @@ -451,7 +451,7 @@ export const loadGeoTIFFWithCache = async (sourceInfo: { const metadata = await Gdal.gdalinfo(tifDataset, ['-stats']); Gdal.close(tifDataset); - await saveToCache(sourceInfo.url, fileBlob, metadata); + await saveToIndexedDB(sourceInfo.url, fileBlob, metadata); return { file: fileBlob, @@ -460,33 +460,6 @@ export const loadGeoTIFFWithCache = async (sourceInfo: { }; }; -const getFromCache = async (key: string) => { - const cachedData = await getFromIndexedDB(key); - if (cachedData) { - const data = - cachedData.file instanceof Blob - ? new Blob([cachedData.file]) - : cachedData.file; - - return { - data, - metadata: cachedData.metadata - }; - } - return null; -}; - -const saveToCache = async ( - key: string, - data: - | Blob - | shp.FeatureCollectionWithFilename - | shp.FeatureCollectionWithFilename[], - metadata?: any -) => { - await saveToIndexedDB(key, data, metadata); -}; - /** * Generalized file reader for different source types. * @@ -524,9 +497,9 @@ export const loadFile = async (fileInfo: { } case 'ShapefileSource': { - const cached = await getFromCache(filepath); + const cached = await getFromIndexedDB(filepath); if (cached) { - return cached.data; + return cached.file; } try { @@ -535,7 +508,7 @@ export const loadFile = async (fileInfo: { ); const arrayBuffer = await response.arrayBuffer(); const geojson = await shp(arrayBuffer); - await saveToCache(filepath, geojson); + await saveToIndexedDB(filepath, geojson); return geojson; } catch (error) { console.error('Error loading remote shapefile:', error); @@ -544,9 +517,9 @@ export const loadFile = async (fileInfo: { } case 'GeoJSONSource': { - const cached = await getFromCache(filepath); + const cached = await getFromIndexedDB(filepath); if (cached) { - return cached.data; + return cached.file; } try { @@ -557,7 +530,7 @@ export const loadFile = async (fileInfo: { throw new Error(`Failed to fetch GeoJSON from URL: ${filepath}`); } const geojson = await response.json(); - await saveToCache( + await saveToIndexedDB( filepath, geojson as shp.FeatureCollectionWithFilename ); From 04a51e3d05681f71ee959dda244aa3918aaaafdd Mon Sep 17 00:00:00 2001 From: Arjun Verma Date: Thu, 23 Jan 2025 15:46:28 +0530 Subject: [PATCH 11/12] Update packages/base/src/tools.ts Co-authored-by: martinRenou --- packages/base/src/tools.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index 4d9fb3f9e..92fb0d908 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -532,7 +532,7 @@ export const loadFile = async (fileInfo: { const geojson = await response.json(); await saveToIndexedDB( filepath, - geojson as shp.FeatureCollectionWithFilename + geojson ); return geojson; } catch (error) { From eb7b08c291d9163796ec45792e8756c2659c29c1 Mon Sep 17 00:00:00 2001 From: arjxn-py Date: Thu, 23 Jan 2025 15:50:12 +0530 Subject: [PATCH 12/12] lint --- packages/base/src/tools.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index 92fb0d908..16539e025 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -406,10 +406,13 @@ export const saveToIndexedDB = async ( */ export const getFromIndexedDB = async (key: string) => { const db = await openDatabase(); - return new Promise<{ - file: any; - metadata?: any | undefined; - } | undefined>((resolve, reject) => { + return new Promise< + | { + file: any; + metadata?: any | undefined; + } + | undefined + >((resolve, reject) => { const transaction = db.transaction('files', 'readonly'); const store = transaction.objectStore('files'); const request = store.get(key); @@ -530,10 +533,7 @@ export const loadFile = async (fileInfo: { throw new Error(`Failed to fetch GeoJSON from URL: ${filepath}`); } const geojson = await response.json(); - await saveToIndexedDB( - filepath, - geojson - ); + await saveToIndexedDB(filepath, geojson); return geojson; } catch (error) { console.error('Error loading remote GeoJSON:', error);