diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 30e5bcd033..1bd06e3a7d 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -1,3 +1,4 @@ +aac Aadil aadl aarondou @@ -10,6 +11,7 @@ ablack abspath abstractmethod ABuffer +abw acconstantsini Accu accurev @@ -25,8 +27,10 @@ ACTIVETEXTLOGGERIMPL actools acwrap acxz +ada addoffset addon +addslash ADEBAD adminlist aeiouy @@ -47,7 +51,6 @@ ANamespace anotherchan ANOTHEREVENT anotherparam -aop apad api APID @@ -74,10 +77,10 @@ args argtype argv Arial +arijitdas arinc arity arpa -asctime asd asdka ASer @@ -85,7 +88,6 @@ ASerialize askjd aslkjd asm -aspectj aspx ASTRING asyc @@ -122,19 +124,16 @@ autoraise autosectionlabel awt AYYYY -arijitdas backend backface backport backslashreplace BADCHECKSUM -BAREMERAL baremetal BARETASKHANDLE baseclass basestring bashcompinit -bashrc Bassic batchmode baudrate @@ -158,6 +157,7 @@ blockquote blog BLSP BLSPSERIALDRIVERCOMPONENTIMPLCFG +bmp bocchino bodychars bodytext @@ -182,7 +182,6 @@ BUFFERTOOSMALL buffsize BUGLIST builddir -Buildfile buildroot buildtargets builtins @@ -193,7 +192,6 @@ BYTEDRV BYTESRECEIVED BYTESRECV BYTESSENT -cae calcsize calcu calibri @@ -210,6 +208,7 @@ cassert cata Catchen CBF +cbl CBLOCK CCB CCFF @@ -254,11 +253,9 @@ Classloader classmethod classname classpath -classpathentry classtype clazz clearfix -clibwrapper clion CLOCAL closedir @@ -298,19 +295,16 @@ columnify combuffer comlogger COMMANDCOMPLETE -COMMANDDISPATCHER COMMANDDISPATCHERIMPL COMMANDDISPATCHERIMPLCFG -commanderror +COMMANDERROR commandline -COMMANDSDISPATCHED COMMANDSEXECUTED commandsi commasepitem commonpath commonprefix COMPACKET -COMPCMDSTAT componentaction COMPONENTTESTERIMPL compositestructures @@ -326,7 +320,6 @@ confparse Connectedoutput Consolas consts -contenttype cooldown coor coravy @@ -357,9 +350,7 @@ crt CRTSCTS cryptsoft CSEQ -csh Csharp -cshrc CSIZE css cstat @@ -382,7 +373,6 @@ curated curating curdir curmsgs -currentxml customise cuz cwd @@ -397,6 +387,7 @@ DASSERT databinding dataobjects datastore +datastructures datetime dawbarton dblclick @@ -437,7 +428,6 @@ dest DEVNULL df dfdc -Dfile dfl DFPRIME DGRAM @@ -475,6 +465,7 @@ docstring doctag doctest doctype +docx doinit Donatas donsim @@ -499,9 +490,6 @@ doxypypy doxyrules doxysearch Doxywizard -DOY -DOYTHH -DPLATFORM DPRIVATE DPROTECTED dropdown @@ -516,7 +504,6 @@ drvtcpclientsocket drvtcpserversocket drvudpsocket dsdl -DSKIP dspal dst DSTATIC @@ -557,7 +544,6 @@ EINPROGRESS EINTR EINVAL EISDIR -ejb elif elist ELOG @@ -627,7 +613,7 @@ expr exprtokens externam extlinks -EXTN +extn exts Fabcdef FADV @@ -688,6 +674,8 @@ fio Firefox firest FIXME +flac +flaskext flist floordiv FLUSHFILE @@ -716,11 +704,9 @@ fprime FPRIMEPROTOCOL fprintf fprofile -fpstyle fptable fptr fputil -freehep fromkeys fromtimestamp frontend @@ -730,6 +716,7 @@ fscanf fstream fstrength fsw +fsx fsync ftest func @@ -738,7 +725,6 @@ fus FWCASSERT gainsboro Gangianpour -gantt gbl gcc gcda @@ -750,7 +736,6 @@ gendoc genex genfile GENHUB -genindex genmakebuilder GENREP genshi @@ -799,28 +784,28 @@ Gnd GNDIF GNUC gnueabihf +gnumeric +Golang google googletest Gorang GPGGA GPINT gpio -graphicsio graphviz grayscale -grep grnd GROUNDINTERFACERULES groupadd groupdict groupmod gse -gson gtags gtest GTestbase gui Guire +gz handcoded hardcoded hardcoding @@ -829,6 +814,7 @@ Harriman's hasattr hashlib hashvalue +hdl hdp Heade HEADERSIZE @@ -898,7 +884,6 @@ ignorables IH iif IJET -imageio imap ime img @@ -942,7 +927,6 @@ ints INTSTARTERROR Inttype INTWAITERROR -INVALIDCOMMAND INVALIDMODE INVALIDRECEIVEMODE invisi @@ -958,10 +942,11 @@ IPHELPER ipp IPPROTO ipriority -IPv +ipv IRUSR IRWXU isabs +isalnum isalpha isdigit isdir @@ -1001,18 +986,10 @@ jasonduley javabuilder javac javadoc -javaee javanature javascript -javassist javax Jax -jaxb -jaxrpc -jboss -jbossall -jbosscx -jbosssx jdk jdperez jdt @@ -1020,33 +997,25 @@ jenkins jenkinsci jf JFile -jhall -jide -jiio -jimi jishii jmi -jmyspell -jna -jnp jobrestrictions joinpath JOption joshuaa +jpe +jpeg jpg jpl jplffs -JRE jsdelivr JSO json jsonable jsonified jsonify -jsr jumbotron junit -Jython kbd keepalive kermit @@ -1125,7 +1094,7 @@ logselect lon longdesc LOQQUEUE -lpg +lowercased Lps lpthread lrt @@ -1149,7 +1118,6 @@ makedirs makefiles makeindex MAKEVAR -MALFORMEDCOMMAND malloc MALLOCALLOCATOR mallocator @@ -1158,7 +1126,6 @@ Managerm Manglapus matchobj mathjax -Mathop maxcountryman maxdepth maxlen @@ -1173,12 +1140,9 @@ mday mdbasiccomponents MDFILE mdinternalstructures -mdk mdkernel mdports mdprofiles -mdr -mdserviceclient mdxml mdzip Mehran @@ -1217,7 +1181,6 @@ mname mngr modbus MODESWITCHED -modindex MOSI MOVEFILE moz @@ -1273,11 +1236,11 @@ netdb Netscape's Neue newloc +newname newroot newself newstring newtio -nexport nfds NGAT nh @@ -1291,7 +1254,6 @@ noapp noargport NOBUFFERS NOCOLOR -nocov NOCTTY nogen nolog @@ -1301,10 +1263,7 @@ nonblock noncomma nondetached NONINFRINGEMENT -NOOPRECEIVED -NOOPSTRINGRECEIVED Nop -nopath noqa normalwidths normpath @@ -1340,11 +1299,13 @@ objclass objdoc objmodule objs -ocl oclc +odf odl odo oflag +oga +ogg okidocki oldeol OMG's @@ -1355,10 +1316,6 @@ onload onreadystatechange OParg OPCODEBASE -OPCODECOMPLETED -OPCODEDISPATCHED -OPCODEERROR -OPCODEREGISTERED opcodes opendir OPENERROR @@ -1382,12 +1339,12 @@ ORhex origfile origstatinfo ortega +OS'es OSAL osascript osaves osets osexc -osgi ostate ostream osubgrouping @@ -1400,7 +1357,6 @@ outdir outout outputfile overridable -OS'es packetization packetized PACKETOUTOFBOUNDS @@ -1432,7 +1388,6 @@ Peet perl PERLMOD pexpect -pgrep php phtml pid @@ -1447,6 +1402,7 @@ pkill pkts plainnat plantuml +plist plugin PLUGINDIR pname @@ -1468,6 +1424,7 @@ PORTSELECTOR PORTSEQUENCESTARTED PORTSOUT posix +posixpath ppandian pport PQueue @@ -1499,7 +1456,6 @@ PRMFILEWRITEERROR PRMIDADDED PRMIDNOTFOUND PRMIDUPDATED -PRMLEDINITSTATE prmname PRMSET probs @@ -1550,7 +1506,6 @@ pyserial pytest PYTHONHOME PYTHONPATH -pytz pyw qch QEvent @@ -1581,7 +1536,7 @@ RATEGROUPSTARTED RATELIMITERTESTER rb RBF -Rce +rce RCHILD rcs Rcv @@ -1615,11 +1570,11 @@ regex regexp relaxng relpath -remoting REMOVEDIRECTORY REMOVEFILE removeit reparse +reqclass reqparse rerendered reserializing @@ -1637,6 +1592,7 @@ RGCYCLESLIPS RGD RGDRV RGMAXTIME +RHEL RHH ridgerun riverbankcomputing @@ -1645,7 +1601,6 @@ rmd rmdir rmtree rmul -rnc rng Roboto rootdir @@ -1656,7 +1611,6 @@ RPIDEMO RPIDEMOCOMPONENTIMPLCFG RPISCHEDCONTEXTS rptr -rsclient RSend rst rstrip @@ -1672,8 +1626,6 @@ runtest rx RXD rxor -RHEL -saaj saddr sadl Saikiran @@ -1690,7 +1642,6 @@ sched SCHEDIN schem schematron -schematypens sclk scm scons @@ -1709,7 +1660,6 @@ sendfile sendline SENDPARTIAL sendto -SEQCMDBUFF seqfile seqgen SEQRUNIN @@ -1781,7 +1731,6 @@ Sinha SIZ sizeb sizeof -slf sloc SMAP Smath @@ -1805,7 +1754,6 @@ Someother SOMEOTHEREVENT someotherpath someparam -somesuch sometask somevalue sommercharles @@ -1851,7 +1799,6 @@ startword staticmethod statvfs stdarg -stdbool stddef stderr stdin @@ -1984,11 +1931,11 @@ textui tfile tflat tfn +tgz thead thepihut Thhmmss thiscol -thisdirdoesnotexist thisfiledoesnotexist throwable thtcp @@ -2048,7 +1995,6 @@ toolbar toolchain tooltip tooltiptext -TOOMANYCOMMANDS topologyapp Torvalds tostring @@ -2062,18 +2008,16 @@ treeview trimwhitespace trinomials truediv -truezip TRUNC truncstring -tsch tsn tstring tt ttype Tuszynski -tw twbs TXD +txz typedef typedef'ed typehints @@ -2139,6 +2083,7 @@ usecond usepackage useradd usermod +uset usleep usr ustr @@ -2153,11 +2098,11 @@ valgrind validator vals valud -valuemax valuemin valuenow ve venv +versionchanged versioning vexc VFILE @@ -2188,13 +2133,14 @@ vxworks VXWORKSLOGASSERT WAITALL watney +wav Wdog weakref webbrowser webified webkit +webp webserver -webservice website werkzeug Werror @@ -2202,7 +2148,6 @@ Wextra whitebox whitelist whitesmoke -whl wiki wikipedia wildcards @@ -2218,7 +2163,6 @@ WRITEERROR writelines WRONLY wrs -wsdl WSL Wstrict Wstringop @@ -2226,7 +2170,6 @@ www wx wxgui Xabcdefx -xalan xapian xcode xdf @@ -2237,7 +2180,6 @@ xhtml xhttp xl xlsx -xmi xml xmlfile xmlns @@ -2246,13 +2188,13 @@ xoff xon xor XPath -xsdlib xsh Xss Xvfb XYZZY yacc yacgen +yaml yml yyyymmdd zlib diff --git a/.lgtm.yml b/.lgtm.yml index 296993f05a..07fe75b4a2 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -28,8 +28,7 @@ extraction: # setup the venv and install packages - "python3 -m venv ./fprime-venv" - ". ./fprime-venv/bin/activate" - - "pip install ./Fw/Python" - - "pip install ./Gds" + - "pip install fprime-tools fprime-gds" configure: command: before_index: diff --git a/Fw/Python/setup.py b/Fw/Python/setup.py index 08a2d00e72..2864934e74 100644 --- a/Fw/Python/setup.py +++ b/Fw/Python/setup.py @@ -8,7 +8,7 @@ # # User Install / Upgrade: # ``` -# pip install --upgrade ./Fw/Python +# pip install --upgrade fprime-tools # ``` # # Developer and Dynamic Installation: @@ -16,7 +16,7 @@ # pip install -e ./Fw/Python # ``` ### - +import os from setuptools import find_packages, setup # Setup a python package using setup-tools. This is a newer (and more recommended) technology @@ -28,8 +28,11 @@ setup( # Basic package information. Describes the package and the data contained inside. This # information should match the F prime description information. #### - name="fprime", - version="1.5.3", + name="fprime-tools", + use_scm_version = { + "root": os.path.join("..",".."), + "relative_to": __file__, + }, license="Apache 2.0 License", description="F Prime Flight Software core data types", long_description=""" @@ -60,13 +63,14 @@ to interact with the data coming from the FSW. # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache 2.0", "Operating System :: Unix", "Operating System :: POSIX", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ], @@ -81,7 +85,7 @@ to interact with the data coming from the FSW. ], extras_require={"dev": ["black==21.5b1", "pylama==7.7.1", "pylint==2.8.2", "pre-commit==2.12.1"]}, # Setup and test requirements, not needed by normal install - setup_requires=["pytest-runner==5.3.0"], + setup_requires=["pytest-runner==5.3.0","setuptools_scm==6.0.1"], tests_require=["pytest"], # Create a set of executable entry-points for running directly from the package entry_points={ diff --git a/Gds/README.md b/Gds/README.md index bb925ce8a6..4554d30851 100644 --- a/Gds/README.md +++ b/Gds/README.md @@ -164,10 +164,7 @@ The Gds requires the packages specified in [setup.py](setup.py). These can be installed along the Gds package using the following commands: ``` -git clone https://github.com/nasa/fprime.git -cd fprime -pip install --upgrade wheel setuptools pip -pip install Gds/ +pip install --upgrade fprime-gds ``` For full installation instructions, including virtual environment creation and installation verification, see [INSTALL.md](https://github.com/nasa/fprime/blob/devel/docs/INSTALL.md). diff --git a/Gds/setup.py b/Gds/setup.py index 7cc97dbc5e..8af8ea405b 100644 --- a/Gds/setup.py +++ b/Gds/setup.py @@ -15,7 +15,7 @@ # # User Install / Upgrade: # ``` -# pip install --upgrade ./Gds +# pip install --upgrade fprime-gds # ``` # # Developer and Dynamic Installation: @@ -44,7 +44,10 @@ setup( # information should match the F prime description information. #### name="fprime_gds", - version="1.5.0", + use_scm_version={ + "root": "..", + "relative_to": __file__ + }, license="Apache 2.0 License", description="F Prime Flight Software Ground Data System layer.", long_description=""" @@ -92,28 +95,25 @@ integrated configuration with ground in-the-loop. # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache 2.0", "Operating System :: Unix", "Operating System :: POSIX", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - # uncomment if you test on these interpreters: - # 'Programming Language :: Python :: Implementation :: IronPython', - # 'Programming Language :: Python :: Implementation :: Jython', - # 'Programming Language :: Python :: Implementation :: Stackless', ], python_requires=">=3.6", + setup_requires=["setuptools_scm==6.0.1"], install_requires=[ "flask==1.1.2", "pexpect==4.8.0", "pytest==6.2.4", "flask_restful==0.3.8", - "fprime>=1.3.0", - "flask_uploads @ git+https://github.com/maxcountryman/flask-uploads@f66d7dc93e684fa0a3a4350a38e41ae00483a796", + "fprime-tools>=1.5.4", "argcomplete==1.12.3", ], extras_require={ diff --git a/Gds/src/fprime_gds/flask/app.py b/Gds/src/fprime_gds/flask/app.py index 03fb93782e..afa460f9c5 100644 --- a/Gds/src/fprime_gds/flask/app.py +++ b/Gds/src/fprime_gds/flask/app.py @@ -11,7 +11,7 @@ import sys import flask import flask_restful -import flask_uploads +from fprime_gds.flask import flask_uploads import fprime_gds.flask.channels diff --git a/Gds/src/fprime_gds/flask/flask_uploads.py b/Gds/src/fprime_gds/flask/flask_uploads.py new file mode 100644 index 0000000000..dae95b2434 --- /dev/null +++ b/Gds/src/fprime_gds/flask/flask_uploads.py @@ -0,0 +1,532 @@ +# -*- coding: utf-8 -*- +""" +flaskext.uploads +================ +This module provides upload support for Flask. The basic pattern is to set up +an `UploadSet` object and upload your files to it. + +:copyright: 2010 Matthew "LeafStorm" Frazier +:license: MIT/X11, see LICENSE for details + +Note: originally from https://github.com/maxcountryman/flask-uploads +""" + +import sys + +PY3 = sys.version_info[0] == 3 + +if PY3: + string_types = str, +else: + string_types = basestring, + +import os.path +import posixpath + +from flask import current_app, send_from_directory, abort, url_for +from itertools import chain # lgtm [py/unused-import] +from werkzeug.datastructures import FileStorage +from werkzeug.utils import secure_filename + +from flask import Blueprint + +# Extension presets + +#: This just contains plain text files (.txt). +TEXT = ('txt',) + +#: This contains various office document formats (.rtf, .odf, .ods, .gnumeric, +#: .abw, .doc, .docx, .xls, .xlsx and .pdf). Note that the macro-enabled versions +#: of Microsoft Office 2007 files are not included. +DOCUMENTS = tuple('rtf odf ods gnumeric abw doc docx xls xlsx pdf'.split()) + +#: This contains basic image types that are viewable from most browsers (.jpg, +#: .jpe, .jpeg, .png, .gif, .svg, .bmp and .webp). +IMAGES = tuple('jpg jpe jpeg png gif svg bmp webp'.split()) + +#: This contains audio file types (.wav, .mp3, .aac, .ogg, .oga, and .flac). +AUDIO = tuple('wav mp3 aac ogg oga flac'.split()) + +#: This is for structured data files (.csv, .ini, .json, .plist, .xml, .yaml, +#: and .yml). +DATA = tuple('csv ini json plist xml yaml yml'.split()) + +#: This contains various types of scripts (.js, .php, .pl, .py .rb, and .sh). +#: If your Web server has PHP installed and set to auto-run, you might want to +#: add ``php`` to the DENY setting. +SCRIPTS = tuple('js php pl py rb sh'.split()) + +#: This contains archive and compression formats (.gz, .bz2, .zip, .tar, +#: .tgz, .txz, and .7z). +ARCHIVES = tuple('gz bz2 zip tar tgz txz 7z'.split()) + +#: This contains nonexecutable source files - those which need to be +#: compiled or assembled to binaries to be used. They are generally safe to +#: accept, as without an existing RCE vulnerability, they cannot be compiled, +#: assembled, linked, or executed. Supports C, C++, Ada, Rust, Go (Golang), +#: FORTRAN, D, Java, C Sharp, F Sharp (compiled only), COBOL, Haskell, and +#: assembly. +SOURCE = tuple(('c cpp c++ h hpp h++ cxx hxx hdl ' # C/C++ + + 'ada ' # Ada + + 'rs ' # Rust + + 'go ' # Go + + 'f for f90 f95 f03 ' # FORTRAN + + 'd dd di ' # D + + 'java ' # Java + + 'hs ' # Haskell + + 'cs ' # C Sharp + + 'fs ' # F Sharp compiled source (NOT .fsx, which is interactive-ready) + + 'cbl cob ' # COBOL + + 'asm s ' # Assembly + ).split()) + +#: This contains shared libraries and executable files (.so, .exe and .dll). +#: Most of the time, you will not want to allow this - it's better suited for +#: use with `AllExcept`. +EXECUTABLES = tuple('so exe dll'.split()) + +#: The default allowed extensions - `TEXT`, `DOCUMENTS`, `DATA`, and `IMAGES`. +DEFAULTS = TEXT + DOCUMENTS + IMAGES + DATA + + +class UploadNotAllowed(Exception): + """ + This exception is raised if the upload was not allowed. You should catch + it in your view code and display an appropriate message to the user. + """ + + +def tuple_from(*iters): + return tuple(itertools.chain(*iters)) + + +def extension(filename): + ext = os.path.splitext(filename)[1] + if ext == '': + # add non-ascii filename support + ext = os.path.splitext(filename)[0] + if ext.startswith('.'): + # os.path.splitext retains . separator + ext = ext[1:] + return ext + + +def lowercase_ext(filename): + """ + This is a helper used by UploadSet.save to provide lowercase extensions for + all processed files, to compare with configured extensions in the same + case. + + .. versionchanged:: 0.1.4 + Filenames without extensions are no longer lowercased, only the + extension is returned in lowercase, if an extension exists. + + :param filename: The filename to ensure has a lowercase extension. + """ + if '.' in filename: + main, ext = os.path.splitext(filename) + return main + ext.lower() + # For consistency with os.path.splitext, + # do not treat a filename without an extension as an extension. + # That is, do not return filename.lower(). + return filename + + +def addslash(url): + if url.endswith('/'): + return url + return url + '/' + + +def patch_request_class(app, size=64 * 1024 * 1024): + """ + By default, Flask will accept uploads to an arbitrary size. While Werkzeug + switches uploads from memory to a temporary file when they hit 500 KiB, + it's still possible for someone to overload your disk space with a + gigantic file. + + This patches the app's request class's + `~werkzeug.BaseRequest.max_content_length` attribute so that any upload + larger than the given size is rejected with an HTTP error. + + .. note:: + + In Flask 0.6, you can do this by setting the `MAX_CONTENT_LENGTH` + setting, without patching the request class. To emulate this behavior, + you can pass `None` as the size (you must pass it explicitly). That is + the best way to call this function, as it won't break the Flask 0.6 + functionality if it exists. + + .. versionchanged:: 0.1.1 + + :param app: The app to patch the request class of. + :param size: The maximum size to accept, in bytes. The default is 64 MiB. + If it is `None`, the app's `MAX_CONTENT_LENGTH` configuration + setting will be used to patch. + """ + if size is None: + if isinstance(app.request_class.__dict__['max_content_length'], + property): + return + size = app.config.get('MAX_CONTENT_LENGTH') + reqclass = app.request_class + patched = type(reqclass.__name__, (reqclass,), + {'max_content_length': size}) + app.request_class = patched + + +def config_for_set(uset, app, defaults=None): + """ + This is a helper function for `configure_uploads` that extracts the + configuration for a single set. + + :param uset: The upload set. + :param app: The app to load the configuration from. + :param defaults: A dict with keys `url` and `dest` from the + `UPLOADS_DEFAULT_DEST` and `DEFAULT_UPLOADS_URL` + settings. + """ + config = app.config + prefix = 'UPLOADED_%s_' % uset.name.upper() + using_defaults = False + if defaults is None: + defaults = dict(dest=None, url=None) + + allow_extns = tuple(config.get(prefix + 'ALLOW', ())) + deny_extns = tuple(config.get(prefix + 'DENY', ())) + destination = config.get(prefix + 'DEST') + base_url = config.get(prefix + 'URL') + + if destination is None: + # the upload set's destination wasn't given + if uset.default_dest: + # use the "default_dest" callable + destination = uset.default_dest(app) + if destination is None: # still + # use the default dest from the config + if defaults['dest'] is not None: + using_defaults = True + destination = os.path.join(defaults['dest'], uset.name) + else: + raise RuntimeError("no destination for set %s" % uset.name) + + if base_url is None and using_defaults and defaults['url']: + base_url = addslash(defaults['url']) + uset.name + '/' + + return UploadConfiguration(destination, base_url, allow_extns, deny_extns) + + +def configure_uploads(app, upload_sets): + """ + Call this after the app has been configured. It will go through all the + upload sets, get their configuration, and store the configuration on the + app. It will also register the uploads module if it hasn't been set. This + can be called multiple times with different upload sets. + + .. versionchanged:: 0.1.3 + The uploads module/blueprint will only be registered if it is needed + to serve the upload sets. + + :param app: The `~flask.Flask` instance to get the configuration from. + :param upload_sets: The `UploadSet` instances to configure. + """ + if isinstance(upload_sets, UploadSet): + upload_sets = (upload_sets,) + + if not hasattr(app, 'upload_set_config'): + app.upload_set_config = {} + set_config = app.upload_set_config + defaults = dict(dest=app.config.get('UPLOADS_DEFAULT_DEST'), + url=app.config.get('UPLOADS_DEFAULT_URL')) + + for uset in upload_sets: + config = config_for_set(uset, app, defaults) + set_config[uset.name] = config + + should_serve = any(s.base_url is None for s in set_config.values()) + if '_uploads' not in app.blueprints and should_serve: + app.register_blueprint(uploads_mod) + + +class All(object): + """ + This type can be used to allow all extensions. There is a predefined + instance named `ALL`. + """ + def __contains__(self, item): + return True + + +#: This "contains" all items. You can use it to allow all extensions to be +#: uploaded. +ALL = All() + + +class AllExcept(object): + """ + This can be used to allow all file types except certain ones. For example, + to ban .exe and .iso files, pass:: + + AllExcept(('exe', 'iso')) + + to the `UploadSet` constructor as `extensions`. You can use any container, + for example:: + + AllExcept(SCRIPTS + EXECUTABLES) + """ + def __init__(self, items): + self.items = items + + def __contains__(self, item): + return item not in self.items + + +class UploadConfiguration(object): + """ + This holds the configuration for a single `UploadSet`. The constructor's + arguments are also the attributes. + + :param destination: The directory to save files to. + :param base_url: The URL (ending with a /) that files can be downloaded + from. If this is `None`, Flask-Uploads will serve the + files itself. + :param allow: A list of extensions to allow, even if they're not in the + `UploadSet` extensions list. + :param deny: A list of extensions to deny, even if they are in the + `UploadSet` extensions list. + """ + def __init__(self, destination, base_url=None, allow=(), deny=()): + self.destination = destination + self.base_url = base_url + self.allow = allow + self.deny = deny + + @property + def tuple(self): + return (self.destination, self.base_url, self.allow, self.deny) + + def __eq__(self, other): + return self.tuple == other.tuple + + +class UploadSet(object): + """ + This represents a single set of uploaded files. Each upload set is + independent of the others. This can be reused across multiple application + instances, as all configuration is stored on the application object itself + and found with `flask.current_app`. + + :param name: The name of this upload set. It defaults to ``files``, but + you can pick any alphanumeric name you want. (For simplicity, + it's best to use a plural noun.) + :param extensions: The extensions to allow uploading in this set. The + easiest way to do this is to add together the extension + presets (for example, ``TEXT + DOCUMENTS + IMAGES``). + It can be overridden by the configuration with the + `UPLOADED_X_ALLOW` and `UPLOADED_X_DENY` configuration + parameters. The default is `DEFAULTS`. + :param default_dest: If given, this should be a callable. If you call it + with the app, it should return the default upload + destination path for that app. + """ + def __init__(self, name='files', extensions=DEFAULTS, default_dest=None): + if not name.isalnum(): + raise ValueError("Name must be alphanumeric (no underscores)") + self.name = name + self.extensions = extensions + self._config = None + self.default_dest = default_dest + + @property + def config(self): + """ + This gets the current configuration. By default, it looks up the + current application and gets the configuration from there. But if you + don't want to go to the full effort of setting an application, or it's + otherwise outside of a request context, set the `_config` attribute to + an `UploadConfiguration` instance, then set it back to `None` when + you're done. + """ + if self._config is not None: + return self._config + try: + return current_app.upload_set_config[self.name] + except AttributeError: + raise RuntimeError("cannot access configuration outside request") + + def url(self, filename): + """ + This function gets the URL a file uploaded to this set would be + accessed at. It doesn't check whether said file exists. + + :param filename: The filename to return the URL for. + """ + base = self.config.base_url + if base is None: + return url_for('_uploads.uploaded_file', setname=self.name, + filename=filename, _external=True) + else: + return base + filename + + def path(self, filename, folder=None): + """ + This returns the absolute path of a file uploaded to this set. It + doesn't actually check whether said file exists. + + :param filename: The filename to return the path for. + :param folder: The subfolder within the upload set previously used + to save to. + """ + if folder is not None: + target_folder = os.path.join(self.config.destination, folder) + else: + target_folder = self.config.destination + return os.path.join(target_folder, filename) + + def file_allowed(self, storage, basename): + """ + This tells whether a file is allowed. It should return `True` if the + given `werkzeug.FileStorage` object can be saved with the given + basename, and `False` if it can't. The default implementation just + checks the extension, so you can override this if you want. + + :param storage: The `werkzeug.FileStorage` to check. + :param basename: The basename it will be saved under. + """ + return self.extension_allowed(extension(basename)) + + def extension_allowed(self, ext): + """ + This determines whether a specific extension is allowed. It is called + by `file_allowed`, so if you override that but still want to check + extensions, call back into this. + + :param ext: The extension to check, without the dot. + """ + return ((ext in self.config.allow) or + (ext in self.extensions and ext not in self.config.deny)) + + def get_basename(self, filename): + return lowercase_ext(secure_filename(filename)) + + def save(self, storage, folder=None, name=None): + """ + This saves a `werkzeug.FileStorage` into this upload set. If the + upload is not allowed, an `UploadNotAllowed` error will be raised. + Otherwise, the file will be saved and its name (including the folder) + will be returned. + + :param storage: The uploaded file to save. + :param folder: The subfolder within the upload set to save to. + :param name: The name to save the file as. If it ends with a dot, the + file's extension will be appended to the end. (If you + are using `name`, you can include the folder in the + `name` instead of explicitly using `folder`, i.e. + ``uset.save(file, name="someguy/photo_123.")`` + """ + if not isinstance(storage, FileStorage): + raise TypeError("storage must be a werkzeug.FileStorage") + + if folder is None and name is not None and "/" in name: + folder, name = os.path.split(name) + + basename = self.get_basename(storage.filename) + + if not self.file_allowed(storage, basename): + raise UploadNotAllowed() + + if name: + if name.endswith('.'): + basename = name + extension(basename) + else: + basename = name + + + + if folder: + target_folder = os.path.join(self.config.destination, folder) + else: + target_folder = self.config.destination + if not os.path.exists(target_folder): + os.makedirs(target_folder) + if os.path.exists(os.path.join(target_folder, basename)): + basename = self.resolve_conflict(target_folder, basename) + + target = os.path.join(target_folder, basename) + storage.save(target) + if folder: + return posixpath.join(folder, basename) + else: + return basename + + def resolve_conflict(self, target_folder, basename): + """ + If a file with the selected name already exists in the target folder, + this method is called to resolve the conflict. It should return a new + basename for the file. + + The default implementation splits the name and extension and adds a + suffix to the name consisting of an underscore and a number, and tries + that until it finds one that doesn't exist. + + :param target_folder: The absolute path to the target. + :param basename: The file's original basename. + """ + name, ext = os.path.splitext(basename) + count = 0 + while True: + count = count + 1 + newname = '%s_%d%s' % (name, count, ext) + if not os.path.exists(os.path.join(target_folder, newname)): + return newname + + +uploads_mod = Blueprint('_uploads', __name__, url_prefix='/_uploads') + + +@uploads_mod.route('//') +def uploaded_file(setname, filename): + config = current_app.upload_set_config.get(setname) + if config is None: + abort(404) + return send_from_directory(config.destination, filename) + + +class TestingFileStorage(FileStorage): + """ + This is a helper for testing upload behavior in your application. You + can manually create it, and its save method is overloaded to set `saved` + to the name of the file it was saved to. All of these parameters are + optional, so only bother setting the ones relevant to your application. + + :param stream: A stream. The default is an empty stream. + :param filename: The filename uploaded from the client. The default is the + stream's name. + :param name: The name of the form field it was loaded from. The default is + `None`. + :param content_type: The content type it was uploaded as. The default is + ``application/octet-stream``. + :param content_length: How long it is. The default is -1. + :param headers: Multipart headers as a `werkzeug.Headers`. The default is + `None`. + """ + def __init__(self, stream=None, filename=None, name=None, + content_type='application/octet-stream', content_length=-1, + headers=None): + FileStorage.__init__(self, stream, filename, name=name, + content_type=content_type, content_length=content_length, + headers=None) + self.saved = None + + def save(self, dst, buffer_size=16384): + """ + This marks the file as saved by setting the `saved` attribute to the + name of the file it was saved to. + + :param dst: The file to save to. + :param buffer_size: Ignored. + """ + if isinstance(dst, string_types): + self.saved = dst + else: + self.saved = dst.name diff --git a/README.md b/README.md index 2a7ec921f3..b9f45b62ba 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,7 @@ To install F´ quickly, enter: ``` git clone https://github.com/nasa/fprime.git -cd fprime -pip install --upgrade wheel setuptools pip -pip install Fw/Python Gds/ +pip install --upgrade fprime-tools fprime-gds ``` For full installation instructions, including virtual environment creation and installation verification, see [INSTALL.md](./docs/INSTALL.md). diff --git a/ci/bootstrap.bash b/ci/bootstrap.bash index 8aea4ce50f..dd349fa794 100755 --- a/ci/bootstrap.bash +++ b/ci/bootstrap.bash @@ -14,8 +14,6 @@ rm -rf "${USABLE_VENV}" python3 -m venv "${USABLE_VENV}" || fail_and_stop "Failed to create VENV" . "${USABLE_VENV}/bin/activate" || fail_and_stop "Failed to source VENV" echo -e "Installing PIP Packages" -pip install -U pip wheel || fail_and_stop "Failed to bootstrap pip" - # install dependencies based on the TEST_TYPE if [[ "${TEST_TYPE}" == "STATIC" ]] then @@ -23,6 +21,5 @@ then pip install -U pylama pylama_pylint radon else # These are required for all other tests - pip install ${FPRIME_DIR}/Fw/Python || fail_and_stop "Failed to install fprime PIP module from ./Fw/Python" - pip install ${FPRIME_DIR}/Gds[test-api-xls] || fail_and_stop "Failed to install fprime PIP module from ./Gds[test-api-xls]" + pip install fprime-tools fprime-gds || fail_and_stop "Failed to install fprime PIP module from ./Fw/Python" fi diff --git a/docker/Dockerfile b/docker/Dockerfile index f8ec29947b..c4ec20ea5c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,9 +36,7 @@ ENTRYPOINT ["/bin/bash"] FROM fprime-base AS fprime-docker RUN git clone --quiet https://github.com/nasa/fprime.git /opt/fprime && \ python3 -m venv /opt/fprime-venv/ && . /opt/fprime-venv/bin/activate && \ - pip install --no-cache-dir -U wheel setuptools pip && \ - pip install --no-cache-dir /opt/fprime/Fw/Python/ && \ - pip install --no-cache-dir /opt/fprime/Gds/ && \ + pip install --no-cache-dir fprime-tools fprime-gds && \ rm -r ~/.cache/pip && \ chown -R fprime:fprime /opt/fprime-venv && \ chmod -R 775 /opt/fprime-venv diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 821950de52..0230c6bde1 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -80,10 +80,7 @@ tools package. This is to enable users to choose which tools they'd like to use. **Installing F´ Python Packages** ``` -pip install --upgrade wheel setuptools pip -cd -pip install ./Fw/Python -pip install ./Gds +pip install --upgrade fprime-tools fprime-gds ``` ## Checking Your F´ Installation diff --git a/docs/index.md b/docs/index.md index eba8b9ee46..dff83c2079 100644 --- a/docs/index.md +++ b/docs/index.md @@ -50,9 +50,7 @@ virtual environment), and building one of our reference applications. For full i ``` git clone https://github.com/nasa/fprime.git -cd fprime -pip install --upgrade wheel setuptools pip -pip install Fw/Python Gds/ +pip install --upgrade fprime-tools fprime-gds ``` **Build the Ref Application**