Skip to content

Commit

Permalink
Add identifier to uploaded file names (#53)
Browse files Browse the repository at this point in the history
Fixes a major bug where the uploaded files in a batch upload all had
the same name. This caused the learning loop to import the same file up
to 20 times.

---------

Co-authored-by: Niklas Neugebauer <[email protected]>
  • Loading branch information
denniswittich and NiklasNeugebauer authored Jan 24, 2025
1 parent 6c1f422 commit 525c9d0
Showing 1 changed file with 11 additions and 5 deletions.
16 changes: 11 additions & 5 deletions learning_loop_node/detector/outbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,10 @@ def save(self,
image_metadata.source = source or 'unknown'
os.makedirs(tmp, exist_ok=True)

with open(tmp + '/image.json', 'w') as f:
with open(tmp + f'/image_{identifier}.json', 'w') as f:
json.dump(jsonable_encoder(asdict(image_metadata)), f)

with open(tmp + '/image.jpg', 'wb') as f:
with open(tmp + f'/image_{identifier}.jpg', 'wb') as f:
f.write(image)

if os.path.exists(tmp):
Expand Down Expand Up @@ -141,8 +141,10 @@ async def _upload_batch(self, items: List[str]):
# results in a post failure on the first run of the test in a docker environment (WTF)

data: List[Tuple[str, Union[TextIOWrapper, BufferedReader]]] = []
data = [('files', open(f'{item}/image.json', 'r')) for item in items]
data += [('files', open(f'{item}/image.jpg', 'rb')) for item in items]
for item in items:
identifier = os.path.basename(item)
data.append(('files', open(f'{item}/image_{identifier}.json', 'r')))
data.append(('files', open(f'{item}/image_{identifier}.jpg', 'rb')))

try:
async with aiohttp.ClientSession() as session:
Expand All @@ -158,7 +160,11 @@ async def _upload_batch(self, items: List[str]):
if response.status == 200:
self.upload_counter += len(items)
for item in items:
shutil.rmtree(item, ignore_errors=True)
try:
shutil.rmtree(item)
self.log.debug('Deleted %s', item)
except Exception:
self.log.exception('Failed to delete %s', item)
self.log.info('Uploaded %s images successfully', len(items))

elif response.status == 422:
Expand Down

0 comments on commit 525c9d0

Please sign in to comment.