diff --git a/api/scripts b/api/scripts index 657a639..16909ea 160000 --- a/api/scripts +++ b/api/scripts @@ -1 +1 @@ -Subproject commit 657a63975325f42bd932108d7d73ddfbd04487a7 +Subproject commit 16909ea356ff6695d3e97d46efce78bca5fe5c40 diff --git a/api/src/paths.js b/api/src/paths.js index 16f12c1..211f743 100644 --- a/api/src/paths.js +++ b/api/src/paths.js @@ -19,6 +19,7 @@ const TRACKS_DIR = path.join(DATA_DIR, 'tracks'); const OBS_FACE_CACHE_DIR = path.join(DATA_DIR, 'obs-face-cache'); module.exports = { + API_ROOT_DIR, DATA_DIR, PROCESSING_DIR, PROCESSING_OUTPUT_DIR, diff --git a/api/src/process_track.py b/api/src/process_track.py index 42013f1..d7d3a1f 100644 --- a/api/src/process_track.py +++ b/api/src/process_track.py @@ -6,7 +6,15 @@ import json from obs.face.importer import ImportMeasurementsCsv from obs.face.annotate import AnnotateMeasurements -from obs.face.filter import PrivacyFilter, ChainFilter, AnonymizationMode, RequiredFieldsFilter, ConfirmedFilter +from obs.face.filter import ( + PrivacyFilter, + ChainFilter, + AnonymizationMode, + RequiredFieldsFilter, + ConfirmedFilter, + PrivacyZone, + PrivacyZonesFilter, +) from obs.face.osm import DataSource as OSMDataSource log = logging.getLogger(__name__) @@ -33,6 +41,12 @@ def main(): dest="cache_dir", help="path where the visualization data will be stored", ) + parser.add_argument( + "--settings", + type=argparse.FileType("rt", encoding="utf-8"), + default=None, + help="path where the visualization data will be stored", + ) args = parser.parse_args() @@ -53,6 +67,9 @@ def process(args): os.makedirs(args.output, exist_ok=True) + log.info("Loading settings") + settings = json.load(args.settings) + log.info("Annotating and filtering CSV file") measurements, statistics = ImportMeasurementsCsv().read( filename_input, @@ -60,16 +77,38 @@ def process(args): dataset_id=dataset_id, ) - measurements = AnnotateMeasurements(osm, cache_dir=args.cache_dir).annotate(measurements) + measurements = AnnotateMeasurements(osm, cache_dir=args.cache_dir).annotate( + measurements + ) - valid_measurements = ChainFilter( + filters_from_settings = [] + for filter_description in settings.get("filters", []): + filter_type = filter_description.get("type") + if filter_type == "PrivacyZonesFilter": + privacy_zones = [ + PrivacyZone( + latitude=zone.get("latitude"), + longitude=zone.get("longitude"), + radius=zone.get("radius"), + ) + for zone in filter_description.get("config", {}).get("privacyZones", []) + ] + filters_from_settings.append(PrivacyZonesFilter(privacy_zones)) + else: + log.warning("Ignoring unknown filter type %r in settings file", filter_type) + + input_filter = ChainFilter( RequiredFieldsFilter(), PrivacyFilter( - user_id_mode=AnonymizationMode.REMOVE, - measurement_id_mode=AnonymizationMode.REMOVE, + user_id_mode=AnonymizationMode.REMOVE, + measurement_id_mode=AnonymizationMode.REMOVE, ), - ).filter(measurements, log=log) - confirmed_measurements = ConfirmedFilter().filter(valid_measurements, log=log) + *filters_from_settings, + ) + confirmed_filter = ConfirmedFilter() + + valid_measurements = input_filter.filter(measurements, log=log) + confirmed_measurements = confirmed_filter.filter(valid_measurements, log=log) # write out confirmed_measurements_json = { @@ -121,14 +160,14 @@ def process(args): } statistics_json = { - "recordedAt": statistics['t_min'].isoformat(), - "recordedUntil": statistics['t_max'].isoformat(), - "duration": statistics['t'], - "length": statistics['d'], - "segments": statistics['n_segments'], - "numEvents": statistics['n_confirmed'], - "numMeasurements": statistics['n_measurements'], - "numValid": statistics['n_valid'], + "recordedAt": statistics["t_min"].isoformat(), + "recordedUntil": statistics["t_max"].isoformat(), + "duration": statistics["t"], + "length": statistics["d"], + "segments": statistics["n_segments"], + "numEvents": statistics["n_confirmed"], + "numMeasurements": statistics["n_measurements"], + "numValid": statistics["n_valid"], } for output_filename, data in [ @@ -137,8 +176,9 @@ def process(args): ("track.json", track_json), ("statistics.json", statistics_json), ]: - with open(os.path.join(args.output, output_filename), 'w') as fp: + with open(os.path.join(args.output, output_filename), "w") as fp: json.dump(data, fp, indent=4) + if __name__ == "__main__": main() diff --git a/api/src/worker.js b/api/src/worker.js index 26647e0..dc852d4 100644 --- a/api/src/worker.js +++ b/api/src/worker.js @@ -39,10 +39,25 @@ queue.process('processTrack', async (job) => { // copy original file to processing dir const inputFilePath = path.join(inputDirectory, 'track.csv'); - const originalFilePath = track.getOriginalFilePath() + const originalFilePath = track.getOriginalFilePath(); console.log(`[${track.slug}] Copy ${originalFilePath} to ${inputFilePath}`); await fs.promises.copyFile(originalFilePath, inputFilePath); + // create track settings file + const settingsFilePath = path.join(inputDirectory, 'track-settings.json'); + console.log(`[${track.slug}] Create settings at ${settingsFilePath}`); + const settings = { + settingsGeneratedAt: new Date().getTime(), + filters: [ + // TODO: Add actual privacy zones from user database + /* { + type: 'PrivacyZonesFilter', + config: { privacyZones: [{ longitude: 10, latitude: 10, radius: 250 }] }, + }, */ + ], + }; + await fs.promises.writeFile(settingsFilePath, JSON.stringify(settings)); + // Create output directory const outputDirectory = path.join(PROCESSING_OUTPUT_DIR, filePath); await fs.promises.mkdir(outputDirectory, { recursive: true }); @@ -54,6 +69,7 @@ queue.process('processTrack', async (job) => { // TODO: Generate track transformation settings (privacy zones etc) // const settingsFilePath = path.join(inputDirectory, 'track-settings.json'); + // const child = spawn( 'python', [ @@ -64,6 +80,8 @@ queue.process('processTrack', async (job) => { outputDirectory, '--path-cache', OBS_FACE_CACHE_DIR, + '--settings', + settingsFilePath, // '--anonymize-user-id', 'remove', // '--anonymize-measurement-id', 'remove', ], @@ -103,7 +121,7 @@ queue.process('processTrack', async (job) => { // Maybe we have found out the recording date, regenerate the automatic // title (if not yet manually set) - await track.autoGenerateTitle() + await track.autoGenerateTitle(); } catch (err) { console.error('Processing failed:', err); track.processingLog += String(err) + '\n' + err.stack + '\n';