diff --git a/meshroom/core/node.py b/meshroom/core/node.py index f2e98a2de3..43aecf8526 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -127,6 +127,101 @@ def fromDict(self, d): self.sessionUid = d.get('sessionUid', '') +class LogManager: + dateTimeFormatting = '%H:%M:%S' + + def __init__(self, chunk): + self.chunk = chunk + self.chunk.statusChanged.connect(self.clear) + self.progressBar = False + self.cleared = False + self.logger = logging.getLogger(chunk.node.getName()) + + class Formatter(logging.Formatter): + def format(self, record): + # Make level name lower case + record.levelname = record.levelname.lower() + return logging.Formatter.format(self, record) + + def configureLogger(self): + for handler in self.logger.handlers[:]: + self.logger.removeHandler(handler) + handler = logging.FileHandler(self.chunk.logFile) + formatter = self.Formatter('[%(asctime)s.%(msecs)03d][%(levelname)s] %(message)s', self.dateTimeFormatting) + handler.setFormatter(formatter) + self.logger.addHandler(handler) + + def clear(self): + if self.chunk.statusName == 'RUNNING' and not self.cleared: + open(self.chunk.logFile, 'w').close() + self.configureLogger() + self.cleared = True + # When the node gets ran again the log needs to be cleared + elif self.chunk.statusName in ['ERROR', 'SUCCESS']: + for handler in self.logger.handlers[:]: + # Stops the file being locked + handler.close() + self.cleared = False + self.progressBar = False + + def waitUntilCleared(self): + while not self.cleared: + time.sleep(0.01) + + def makeProgressBar(self, end, message=''): + assert end > 0 + assert not self.progressBar + self.waitUntilCleared() + + self.progressEnd = end + self.currentProgressTics = 0 + self.progressBar = True + + with open(self.chunk.logFile, 'a') as f: + if message: + f.write(message+'\n') + f.write('0% 10 20 30 40 50 60 70 80 90 100%\n') + f.write('|----|----|----|----|----|----|----|----|----|----|\n\n') + + f.close() + + with open(self.chunk.logFile, 'r') as f: + content = f.read() + self.progressBarPosition = content.rfind('\n') + + f.close() + + def updateProgressBar(self, value): + assert self.progressBar + assert value <= self.progressEnd + self.waitUntilCleared() + + tics = round((value/self.progressEnd)*51) + + with open(self.chunk.logFile, 'r+') as f: + text = f.read() + for i in range(tics-self.currentProgressTics): + text = text[:self.progressBarPosition]+'*'+text[self.progressBarPosition:] + f.seek(0) + f.write(text) + f.close() + + self.currentProgressTics = tics + + def completeProgressBar(self): + assert self.progressBar + + self.progressBar = False + + def textToLevel(self, text): + if text == 'critical': return logging.CRITICAL + elif text == 'error': return logging.ERROR + elif text == 'warning': return logging.WARNING + elif text == 'info': return logging.INFO + elif text == 'debug': return logging.DEBUG + else: return logging.NOTSET + + runningProcesses = {} @@ -142,6 +237,7 @@ def __init__(self, node, range, parent=None): super(NodeChunk, self).__init__(parent) self.node = node self.range = range + self.logManager = LogManager(self) self.status = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion) self.statistics = stats.Statistics() self.statusFileLastModTime = -1 @@ -164,6 +260,10 @@ def name(self): def statusName(self): return self.status.status.name + @property + def logger(self): + return self.logManager.logger + @property def execModeName(self): return self.status.execMode.name diff --git a/meshroom/nodes/aliceVision/Publish.py b/meshroom/nodes/aliceVision/Publish.py index 966566e69c..ebe2b9b832 100644 --- a/meshroom/nodes/aliceVision/Publish.py +++ b/meshroom/nodes/aliceVision/Publish.py @@ -1,6 +1,6 @@ from __future__ import print_function -__version__ = "1.1" +__version__ = "1.2" from meshroom.core import desc import shutil @@ -30,6 +30,15 @@ class Publish(desc.Node): description="", value="", uid=[0], + ), + desc.ChoiceParam( + name='verboseLevel', + label='Verbose Level', + description='''verbosity level (critical, error, warning, info, debug).''', + value='info', + values=['critical', 'error', 'warning', 'info', 'debug'], + exclusive=True, + uid=[], ), ] @@ -41,9 +50,11 @@ def resolvedPaths(self, inputFiles, outDir): return paths def processChunk(self, chunk): - print("Publish") + chunk.logManager.waitUntilCleared() + chunk.logger.setLevel(chunk.logManager.textToLevel(chunk.node.verboseLevel.value)) + if not chunk.node.inputFiles: - print("Nothing to publish") + chunk.logger.warning('Nothing to publish') return if not chunk.node.output.value: return @@ -51,13 +62,15 @@ def processChunk(self, chunk): outFiles = self.resolvedPaths(chunk.node.inputFiles.value, chunk.node.output.value) if not outFiles: - raise RuntimeError("Publish: input files listed, but nothing to publish. " - "Listed input files: {}".format(chunk.node.inputFiles.value)) + error = 'Publish: input files listed, but nothing to publish' + chunk.logger.error(error) + chunk.logger.info('Listed input files: {}'.format([i.value for i in chunk.node.inputFiles.value])) + raise RuntimeError(error) if not os.path.exists(chunk.node.output.value): os.mkdir(chunk.node.output.value) for iFile, oFile in outFiles.items(): - print('Publish file', iFile, 'into', oFile) + chunk.logger.info('Publish file {} into {}'.format(iFile, oFile)) shutil.copyfile(iFile, oFile) - print('Publish end') + chunk.logger.info('Publish end') diff --git a/meshroom/nodes/aliceVision/SketchfabUpload.py b/meshroom/nodes/aliceVision/SketchfabUpload.py new file mode 100644 index 0000000000..2a552b879c --- /dev/null +++ b/meshroom/nodes/aliceVision/SketchfabUpload.py @@ -0,0 +1,280 @@ +__version__ = "1.0" + +from meshroom.core import desc +import glob +import os +import json +import zipfile +import requests +import io + + +class BufferReader(io.BytesIO): # object to call the callback while the file is being uploaded + def __init__(self, buf=b'', + callback=None, + cb_args=(), + cb_kwargs={}, + stopped=None): + self._callback = callback + self._cb_args = cb_args + self._cb_kwargs = cb_kwargs + self._stopped = stopped + self._progress = 0 + self._len = len(buf) + io.BytesIO.__init__(self, buf) + + def __len__(self): + return self._len + + def read(self, n=-1): + chunk = io.BytesIO.read(self, n) + self._progress += int(len(chunk)) + self._cb_kwargs.update({ + 'size' : self._len, + 'progress': self._progress + }) + if self._callback: + try: + self._callback(*self._cb_args, **self._cb_kwargs) + except Exception as e: # catches exception from the callback + self._cb_kwargs['logManager'].logger.warning('Error at callback: {}'.format(e)) + + if self._stopped(): + raise RuntimeError('Node stopped by user') + return chunk + +def progressUpdate(size=None, progress=None, logManager=None): + if not logManager.progressBar: + logManager.makeProgressBar(size, 'Upload progress:') + + logManager.updateProgressBar(progress) + +class SketchfabUpload(desc.Node): + size = desc.DynamicNodeSize('inputFiles') + inputs = [ + desc.ListAttribute( + elementDesc=desc.File( + name="input", + label="Input", + description="", + value="", + uid=[0], + ), + name="inputFiles", + label="Input Files", + description="Input Files to export.", + group="", + ), + desc.StringParam( + name='apiToken', + label='API Token', + description='Get your token from https://sketchfab.com/settings/password', + value='', + uid=[0], + ), + desc.StringParam( + name='title', + label='Title', + description='Title cannot be longer than 48 characters.', + value='', + uid=[0], + ), + desc.StringParam( + name='description', + label='Description', + description='Description cannot be longer than 1024 characters.', + value='', + uid=[0], + ), + desc.ChoiceParam( + name='license', + label='License', + description='License label.', + value='CC Attribution', + values=['CC Attribution', + 'CC Attribution-ShareAlike', + 'CC Attribution-NoDerivs', + 'CC Attribution-NonCommercial', + 'CC Attribution-NonCommercial-ShareAlike', + 'CC Attribution-NonCommercial-NoDerivs'], + exclusive=True, + uid=[0], + ), + desc.ListAttribute( + elementDesc=desc.StringParam( + name='tag', + label='Tag', + description='Tag cannot be longer than 48 characters.', + value='', + uid=[0], + ), + name="tags", + label="Tags", + description="Maximum of 42 separate tags.", + group="", + ), + desc.ChoiceParam( + name='category', + label='Category', + description='Adding categories helps improve the discoverability of your model.', + value='none', + values=['none', + 'animals-pets', + 'architecture', + 'art-abstract', + 'cars-vehicles', + 'characters-creatures', + 'cultural-heritage-history', + 'electronics-gadgets', + 'fashion-style', + 'food-drink', + 'furniture-home', + 'music', + 'nature-plants', + 'news-politics', + 'people', + 'places-travel', + 'science-technology', + 'sports-fitness', + 'weapons-military'], + exclusive=True, + uid=[0], + ), + desc.BoolParam( + name='isPublished', + label='Publish', + description='If the model is not published it will be saved as a draft.', + value=False, + uid=[0], + ), + desc.BoolParam( + name='isInspectable', + label='Inspectable', + description='Allow 2D view in model inspector.', + value=True, + uid=[0], + ), + desc.BoolParam( + name='isPrivate', + label='Private', + description='Requires a pro account.', + value=False, + uid=[0], + ), + desc.StringParam( + name='password', + label='Password', + description='Requires a pro account.', + value='', + uid=[0], + ), + desc.ChoiceParam( + name='verboseLevel', + label='Verbose Level', + description='''verbosity level (critical, error, warning, info, debug).''', + value='info', + values=['critical', 'error', 'warning', 'info', 'debug'], + exclusive=True, + uid=[], + ), + ] + + def upload(self, apiToken, modelFile, data, chunk): + modelEndpoint = 'https://api.sketchfab.com/v3/models' + f = open(modelFile, 'rb') + file = {'modelFile': (os.path.basename(modelFile), f.read())} + file.update(data) + f.close() + (files, contentType) = requests.packages.urllib3.filepost.encode_multipart_formdata(file) + headers = {'Authorization': 'Token {}'.format(apiToken), 'Content-Type': contentType} + body = BufferReader(files, progressUpdate, cb_kwargs={'logManager': chunk.logManager}, stopped=self.stopped) + chunk.logger.info('Uploading...') + try: + r = requests.post( + modelEndpoint, **{'data': body, 'headers': headers}) + chunk.logManager.completeProgressBar() + except requests.exceptions.RequestException as e: + chunk.logger.error(u'An error occured: {}'.format(e)) + raise RuntimeError() + if r.status_code != requests.codes.created: + chunk.logger.error(u'Upload failed with error: {}'.format(r.json())) + raise RuntimeError() + + def resolvedPaths(self, inputFiles): + paths = [] + for inputFile in inputFiles: + if os.path.isdir(inputFile.value): + for path, subdirs, files in os.walk(inputFile.value): + for name in files: + paths.append(os.path.join(path, name)) + else: + for f in glob.glob(inputFile.value): + paths.append(f) + return paths + + def stopped(self): + return self._stopped + + def processChunk(self, chunk): + self._stopped = False + chunk.logManager.waitUntilCleared() + chunk.logger.setLevel(chunk.logManager.textToLevel(chunk.node.verboseLevel.value)) + + if not chunk.node.inputFiles: + chunk.logger.warning('Nothing to upload') + return + if chunk.node.apiToken.value == '': + chunk.logger.error('Need API token.') + raise RuntimeError() + if len(chunk.node.title.value) > 48: + chunk.logger.error('Title cannot be longer than 48 characters.') + raise RuntimeError() + if len(chunk.node.description.value) > 1024: + chunk.logger.error('Description cannot be longer than 1024 characters.') + raise RuntimeError() + tags = [ i.value.replace(' ', '-') for i in chunk.node.tags.value.values() ] + if all(len(i) > 48 for i in tags) and len(tags) > 0: + chunk.logger.error('Tags cannot be longer than 48 characters.') + raise RuntimeError() + if len(tags) > 42: + chunk.logger.error('Maximum of 42 separate tags.') + raise RuntimeError() + + try: + data = { + 'name': chunk.node.title.value, + 'description': chunk.node.description.value, + 'license': chunk.node.license.value, + 'tags': str(tags), + 'isPublished': chunk.node.isPublished.value, + 'isInspectable': chunk.node.isInspectable.value, + 'private': chunk.node.isPrivate.value, + 'password': chunk.node.password.value + } + if chunk.node.category.value != 'none': + data.update({'categories': chunk.node.category.value}) + chunk.logger.debug('Data to be sent: {}'.format(str(data))) + + # pack files into .zip to reduce file size and simplify process + uploadFile = os.path.join(chunk.node.internalFolder, 'temp.zip') + files = self.resolvedPaths(chunk.node.inputFiles.value) + zf = zipfile.ZipFile(uploadFile, 'w') + for file in files: + zf.write(file, os.path.basename(file)) + zf.close() + chunk.logger.debug('Files added to zip: {}'.format(str(files))) + chunk.logger.debug('Created {}'.format(uploadFile)) + chunk.logger.info('File size: {}MB'.format(round(os.path.getsize(uploadFile)/(1024*1024), 3))) + + self.upload(chunk.node.apiToken.value, uploadFile, data, chunk) + chunk.logger.info('Upload successful. Your model is being processed on Sketchfab. It may take some time to show up on your "models" page.') + except Exception as e: + chunk.logger.error(e) + raise RuntimeError() + finally: + if os.path.isfile(uploadFile): + os.remove(uploadFile) + chunk.logger.debug('Deleted {}'.format(uploadFile)) + + def stopProcess(self, chunk): + self._stopped = True diff --git a/requirements.txt b/requirements.txt index 7daba766c1..9caca2201d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ psutil>=5.6.3 enum34;python_version<"3.4" PySide2==5.13.0 markdown==2.6.11 +requests==2.22.0