Skip to content

Commit 1465a92

Browse files
ViBiOhViBiOh
committed
fix: Fixing direct access behavior for vith, and cloning for exas
Signed-off-by: Vincent Boutour <[email protected]>
1 parent 50b203f commit 1465a92

File tree

5 files changed

+66
-40
lines changed

5 files changed

+66
-40
lines changed

‎README.md

+2
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,8 @@ Usage of fibr:
182182
[exif] Aggregate EXIF data per folder on start {FIBR_EXIF_AGGREGATE_ON_START}
183183
-exifDateOnStart
184184
[exif] Change file date from EXIF date on start {FIBR_EXIF_DATE_ON_START}
185+
-exifDirectAccess
186+
[exif] Use Exas with direct access to filesystem (no large file upload to it, send a GET request) {FIBR_EXIF_DIRECT_ACCESS}
185187
-exifGeocodeURL string
186188
[exif] Nominatim Geocode Service URL. This can leak GPS metadatas to a third-party (e.g. "https://nominatim.openstreetmap.org") {FIBR_EXIF_GEOCODE_URL}
187189
-exifMaxSize int

‎docker-compose.example.yaml

+18-6
Original file line numberDiff line numberDiff line change
@@ -9,29 +9,41 @@ services:
99
FIBR_AUTH_USERS: '${BASIC_USERS}'
1010
FIBR_IGNORE_PATTERN: '.st(folder|ignore)'
1111
FIBR_LOGGER_JSON: 'true'
12+
FIBR_EXIF_DIRECT_ACCESS: 'true'
13+
FIBR_THUMBNAIL_DIRECT_ACCESS: 'true'
14+
FIBR_EXIF_MAX_SIZE: '0'
15+
FIBR_THUMBNAIL_MAX_SIZE: '0'
1216
volumes:
1317
- ${DATA_DIR}:/data
1418
depends_on:
1519
- image
1620
- video
21+
- exas
1722
user: '${DATA_USER_ID}'
1823
restart: on-failure:5
1924
read_only: true
2025

26+
image:
27+
image: h2non/imaginary
28+
restart: on-failure:5
29+
read_only: true
30+
2131
video:
2232
image: vibioh/vith
2333
environment:
2434
VITH_LOGGER_JSON: 'true'
35+
VITH_WORK_DIR: '/data'
36+
volumes:
37+
- ${DATA_DIR}:/data
38+
user: '${DATA_USER_ID}'
2539
restart: on-failure:5
2640

27-
image:
28-
image: h2non/imaginary
29-
restart: on-failure:5
30-
read_only: true
31-
3241
exas:
3342
image: vibioh/exas
3443
environment:
3544
EXAS_LOGGER_JSON: 'true'
45+
EXAS_WORK_DIR: '/data'
46+
volumes:
47+
- ${DATA_DIR}:/data
48+
user: '${DATA_USER_ID}'
3649
restart: on-failure:5
37-
read_only: true

‎pkg/exif/exif.go

+39-27
Original file line numberDiff line numberDiff line change
@@ -66,22 +66,25 @@ type App struct {
6666
exifURL string
6767
geocodeURL string
6868
maxSize int64
69-
dateOnStart bool
7069
aggregateOnStart bool
70+
dateOnStart bool
71+
directAccess bool
7172
}
7273

7374
// Config of package
7475
type Config struct {
7576
exifURL *string
7677
geocodeURL *string
7778
maxSize *int
78-
dateOnStart *bool
7979
aggregateOnStart *bool
80+
dateOnStart *bool
81+
directAccess *bool
8082
}
8183

8284
// Flags adds flags for configuring package
8385
func Flags(fs *flag.FlagSet, prefix string, overrides ...flags.Override) Config {
8486
return Config{
87+
directAccess: flags.New(prefix, "exif", "DirectAccess").Default(false, nil).Label("Use Exas with direct access to filesystem (no large file upload to it, send a GET request)").ToBool(fs),
8588
exifURL: flags.New(prefix, "exif", "URL").Default("http://exas:1080", overrides).Label("Exif Tool URL (exas)").ToString(fs),
8689
geocodeURL: flags.New(prefix, "exif", "GeocodeURL").Default("", overrides).Label(fmt.Sprintf("Nominatim Geocode Service URL. This can leak GPS metadatas to a third-party (e.g. \"%s\")", publicNominatimURL)).ToString(fs),
8790
dateOnStart: flags.New(prefix, "exif", "DateOnStart").Default(false, nil).Label("Change file date from EXIF date on start").ToBool(fs),
@@ -102,6 +105,7 @@ func New(config Config, storageApp provider.Storage, prometheusRegisterer promet
102105
geocodeURL: strings.TrimSpace(*config.geocodeURL),
103106
dateOnStart: *config.dateOnStart,
104107
aggregateOnStart: *config.aggregateOnStart,
108+
directAccess: *config.directAccess,
105109
maxSize: int64(*config.maxSize),
106110

107111
storageApp: storageApp,
@@ -149,7 +153,36 @@ func (a App) get(item provider.StorageItem) (map[string]interface{}, error) {
149153
}
150154

151155
func (a App) fetchAndStoreExif(item provider.StorageItem) (map[string]interface{}, error) {
152-
file, err := a.storageApp.ReaderFrom(item.Pathname) // file will be closed by `PipedWriter`
156+
a.increaseMetric("exif", "requested")
157+
158+
resp, err := a.requestExas(context.Background(), item)
159+
if err != nil {
160+
return nil, fmt.Errorf("unable to request exif: %s", err)
161+
}
162+
163+
var exifs []map[string]interface{}
164+
if err := httpjson.Read(resp, &exifs); err != nil {
165+
return nil, fmt.Errorf("unable to read exas response: %s", err)
166+
}
167+
168+
var data map[string]interface{}
169+
if len(exifs) > 0 {
170+
data = exifs[0]
171+
}
172+
173+
if err := a.saveMetadata(item, exifMetadataFilename, data); err != nil {
174+
return nil, fmt.Errorf("unable to save exif: %s", err)
175+
}
176+
177+
return data, nil
178+
}
179+
180+
func (a App) requestExas(ctx context.Context, item provider.StorageItem) (*http.Response, error) {
181+
if a.directAccess {
182+
return request.New().Get(fmt.Sprintf("%s%s", a.exifURL, item.Pathname)).Send(ctx, nil)
183+
}
184+
185+
file, err := a.storageApp.ReaderFrom(item.Pathname) // will be closed by `.PipedWriter`
153186
if err != nil {
154187
return nil, fmt.Errorf("unable to get reader: %s", err)
155188
}
@@ -160,39 +193,18 @@ func (a App) fetchAndStoreExif(item provider.StorageItem) (map[string]interface{
160193
defer provider.BufferPool.Put(buffer)
161194

162195
if _, err := io.CopyBuffer(writer, file, buffer.Bytes()); err != nil {
163-
logger.Error("unable to copy video: %s", err)
196+
logger.Error("unable to copy file: %s", err)
164197
}
165198

166199
_ = writer.CloseWithError(file.Close())
167200
}()
168201

169-
r, err := request.New().WithClient(exasClient).Post(a.exifURL).Build(context.Background(), reader)
202+
r, err := request.New().Post(a.exifURL).Build(ctx, reader)
170203
if err != nil {
171204
return nil, fmt.Errorf("unable to create request: %s", err)
172205
}
173206

174207
r.ContentLength = item.Size
175208

176-
a.increaseMetric("exif", "requested")
177-
178-
resp, err := request.DoWithClient(exasClient, r)
179-
if err != nil {
180-
return nil, err
181-
}
182-
183-
var exifs []map[string]interface{}
184-
if err := httpjson.Read(resp, &exifs); err != nil {
185-
return nil, fmt.Errorf("unable to read exas response: %s", err)
186-
}
187-
188-
var data map[string]interface{}
189-
if len(exifs) > 0 {
190-
data = exifs[0]
191-
}
192-
193-
if err := a.saveMetadata(item, exifMetadataFilename, data); err != nil {
194-
return nil, fmt.Errorf("unable to save exif: %s", err)
195-
}
196-
197-
return data, nil
209+
return request.DoWithClient(exasClient, r)
198210
}

‎pkg/exif/exif_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ func TestFlags(t *testing.T) {
1313
}{
1414
{
1515
"simple",
16-
"Usage of simple:\n -aggregateOnStart\n \t[exif] Aggregate EXIF data per folder on start {SIMPLE_AGGREGATE_ON_START}\n -dateOnStart\n \t[exif] Change file date from EXIF date on start {SIMPLE_DATE_ON_START}\n -geocodeURL string\n \t[exif] Nominatim Geocode Service URL. This can leak GPS metadatas to a third-party (e.g. \"https://nominatim.openstreetmap.org\") {SIMPLE_GEOCODE_URL}\n -maxSize int\n \t[exif] Max file size (in bytes) for extracting exif (0 to no limit) {SIMPLE_MAX_SIZE} (default 209715200)\n -uRL string\n \t[exif] Exif Tool URL (exas) {SIMPLE_URL} (default \"http://exas:1080\")\n",
16+
"Usage of simple:\n -aggregateOnStart\n \t[exif] Aggregate EXIF data per folder on start {SIMPLE_AGGREGATE_ON_START}\n -dateOnStart\n \t[exif] Change file date from EXIF date on start {SIMPLE_DATE_ON_START}\n -directAccess\n \t[exif] Use Exas with direct access to filesystem (no large file upload to it, send a GET request) {SIMPLE_DIRECT_ACCESS}\n -geocodeURL string\n \t[exif] Nominatim Geocode Service URL. This can leak GPS metadatas to a third-party (e.g. \"https://nominatim.openstreetmap.org\") {SIMPLE_GEOCODE_URL}\n -maxSize int\n \t[exif] Max file size (in bytes) for extracting exif (0 to no limit) {SIMPLE_MAX_SIZE} (default 209715200)\n -uRL string\n \t[exif] Exif Tool URL (exas) {SIMPLE_URL} (default \"http://exas:1080\")\n",
1717
},
1818
}
1919

‎pkg/thumbnail/generate.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func (a App) generate(item provider.StorageItem) error {
4040
if item.IsVideo() {
4141
resp, err = a.requestVith(ctx, item)
4242
if err != nil {
43-
return err
43+
return fmt.Errorf("unable to request video thumbnailer: %s", err)
4444
}
4545

4646
file = resp.Body
@@ -85,12 +85,12 @@ func (a App) generate(item provider.StorageItem) error {
8585

8686
func (a App) requestVith(ctx context.Context, item provider.StorageItem) (*http.Response, error) {
8787
if a.directAccess {
88-
return request.New().Get(item.Pathname).Send(ctx, nil)
88+
return request.New().Get(fmt.Sprintf("%s%s", a.videoURL, item.Pathname)).Send(ctx, nil)
8989
}
9090

9191
file, err := a.storageApp.ReaderFrom(item.Pathname) // will be closed by `.PipedWriter`
9292
if err != nil {
93-
return nil, fmt.Errorf("unable to get reader from storage: %s", err)
93+
return nil, fmt.Errorf("unable to get reader: %s", err)
9494
}
9595

9696
reader, writer := io.Pipe()
@@ -99,15 +99,15 @@ func (a App) requestVith(ctx context.Context, item provider.StorageItem) (*http.
9999
defer provider.BufferPool.Put(buffer)
100100

101101
if _, err := io.CopyBuffer(writer, file, buffer.Bytes()); err != nil {
102-
logger.Error("unable to copy video file: %s", err)
102+
logger.Error("unable to copy file: %s", err)
103103
}
104104

105105
_ = writer.CloseWithError(file.Close())
106106
}()
107107

108-
r, err := request.New().WithClient(thumbnailClient).Post(fmt.Sprintf("%s/", a.videoURL)).Build(ctx, reader)
108+
r, err := request.New().Post(a.videoURL).Build(ctx, reader)
109109
if err != nil {
110-
return nil, fmt.Errorf("unable to create video request: %s", err)
110+
return nil, fmt.Errorf("unable to create request: %s", err)
111111
}
112112

113113
r.ContentLength = item.Size

0 commit comments

Comments
 (0)