diff --git a/.travis.yml b/.travis.yml index 422e41d..5b99a41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,7 +74,7 @@ script: fi - if [[ $TEST_TARGET == 'coding_standards' ]]; then - find . -type f -name "*.py" ! -name 'conf.py' ! -name '_version.py' ! -name 'versioneer.py' | xargs flake8 --max-line-length=110 ; + find . -type f -name "*.py" ! -name 'conf.py' ! -name '_version.py' ! -name 'versioneer.py' ! -name '*DBConfig.py' ! -name 'Sample.py' | xargs flake8 --max-line-length=110 ; fi - if [[ $TEST_TARGET == 'docs' ]]; then diff --git a/Examples/Sample.py b/Examples/Sample.py index dcd8896..891b764 100644 --- a/Examples/Sample.py +++ b/Examples/Sample.py @@ -17,19 +17,13 @@ #connect to database # createconnection (dbtype, servername, dbname, username, password) # session_factory = dbconnection.createConnection('connection type: sqlite|mysql|mssql|postgresql', '/your/path/to/db/goes/here', 2.0)#sqlite -session_factory = dbconnection.createConnection('postgresql', 'localhost', 'odm2', 'ODM', 'odm') -# session_factory = dbconnection.createConnection('mysql', 'localhost', 'odm2', 'ODM', 'odm')#mysql -# session_factory= dbconnection.createConnection('mssql', "(local)", "ODM2", "ODM", "odm")#win MSSQL +# session_factory = dbconnection.createConnection('postgresql', 'localhost', 'odm2', 'ODM', 'odm') +# session_factory = dbconnection.createConnection('mysql', 'localhost', 'odm2', 'ODM', 'odm')#mysql +session_factory= dbconnection.createConnection('mssql', "(local)", "ODM2", "ODM", "odm")#win MSSQL # session_factory= dbconnection.createConnection('mssql', "arroyoodm2", "", "ODM", "odm")#mac/linux MSSQL -# session_factory = dbconnection.createConnection('sqlite', '/Users/stephanie/DEV/YODA-Tools/tests/test_files/XL_specimen.sqlite', 2.0) - - - - - - +# session_factory = dbconnection.createConnection('sqlite', 'path/to/ODM2.sqlite', 2.0) @@ -67,6 +61,7 @@ try: print ("\n-------- Information about Sites ---------") siteFeatures = read.getSamplingFeatures(type= 'site') + # siteFeatures = read.getSamplingFeatures(type='Site') numSites = len(siteFeatures) print ("Successful query") @@ -150,7 +145,7 @@ # Get the values for a particular TimeSeriesResult print("\n-------- Example of Retrieving Time Series Result Values ---------") -tsValues = read.getResultValues(resultid = 1) # Return type is a pandas datafram +tsValues = read.getResultValues(resultids = [1]) # Return type is a pandas datafram # Print a few Time Series Values to the console # tsValues.set_index('ValueDateTime', inplace=True) diff --git a/odm2api/ODM2/models.py b/odm2api/ODM2/models.py index 7b6d5b2..5f8c4ac 100644 --- a/odm2api/ODM2/models.py +++ b/odm2api/ODM2/models.py @@ -13,6 +13,9 @@ BigIntegerType = BigIntegerType.with_variant(postgresql.BIGINT(), 'postgresql') BigIntegerType = BigIntegerType.with_variant(mysql.BIGINT(), 'mysql') +DateTimeType = DateTime() +DateTimeType = DateTimeType.with_variant(sqlite.INTEGER(), 'sqlite') + def is_hex(s): try: @@ -25,7 +28,7 @@ def is_hex(s): ################################################################################ # CV ################################################################################ -class CV (object): +class CV(object): __table_args__ = {u'schema': 'odm2'} Term = Column('term', String(255), nullable=False) @@ -181,7 +184,6 @@ class Organizations(Base): class Affiliations(Base): - AffiliationID = Column('affiliationid', Integer, primary_key=True, nullable=False) PersonID = Column('personid', ForeignKey(People.PersonID), nullable=False) OrganizationID = Column('organizationid', ForeignKey(Organizations.OrganizationID)) @@ -230,7 +232,6 @@ class Actions(Base): class ActionBy(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ActionID = Column('actionid', Integer, ForeignKey(Actions.ActionID), nullable=False) AffiliationID = Column('affiliationid', ForeignKey(Affiliations.AffiliationID), nullable=False) @@ -262,22 +263,22 @@ class SamplingFeatures(Base): index=True) """str: Dimensionality of SamplingFeature; point2d, line2d, etc.""" Elevation_m = Column('elevation_m', Float(53)) - """float: The elevation of the sampling feature in meters, or in the case of Specimen, - the elevation from where the SamplingFeature.Specimen was collected""" + """float: The elevation of the sampling feature in meters, or in the case of Specimen, + the elevation from where the SamplingFeature.Specimen was collected""" # noqa ElevationDatumCV = Column('elevationdatumcv', ForeignKey(CVElevationDatum.Name), index=True) """str: The code for the vertical geodetic datum that specifies the zero point for - the Sampling Feature Elevation""" + the Sampling Feature Elevation""" # noqa # FeatureGeometry = Column('featuregeometry', String(50)) """object: The location geometry of the sampling feature on the Earth expressed using a geometry data type. Can be a Point, Curve (profile, trajectory, etc), Surface (flat polygons, etc) or Solid/Volume (although often limited to - 2D geometries). """ + 2D geometries). """ # noqa FeatureGeometryWKT = Column('featuregeometrywkt', String(50)) """str: The location geometry of the sampling feature on the Earth expressed as well known text (WKT). Can be a Point, Curve (profile, trajectory, etc.), Surface (flat polygons, etc.), or Solid/Volume (although often limited to - 2D geometries).""" + 2D geometries).""" # noqa __mapper_args__ = { 'polymorphic_on': case( [ @@ -404,9 +405,9 @@ class Results(Base): ProcessingLevelID = Column('processinglevelid', ForeignKey(ProcessingLevels.ProcessingLevelID), nullable=False) ResultDateTime = Column('resultdatetime', DateTime) - ResultDateTimeUTCOffset = Column('resultdatetimeutcoffset', BigInteger) + ResultDateTimeUTCOffset = Column('resultdatetimeutcoffset', BigIntegerType) ValidDateTime = Column('validdatetime', DateTime) - ValidDateTimeUTCOffset = Column('validdatetimeutcoffset', BigInteger) + ValidDateTimeUTCOffset = Column('validdatetimeutcoffset', BigIntegerType) StatusCV = Column('statuscv', ForeignKey(CVStatus.Name), index=True) SampledMediumCV = Column('sampledmediumcv', ForeignKey(CVMediumType.Name), nullable=False, index=True) ValueCount = Column('valuecount', Integer, nullable=False) @@ -441,7 +442,6 @@ class Results(Base): class DataLoggerProgramFiles(Base): - ProgramID = Column('programid', Integer, primary_key=True, nullable=False) AffiliationID = Column('affiliationid', Integer, ForeignKey(Affiliations.AffiliationID), nullable=False) ProgramName = Column('programname', String(255), nullable=False) @@ -453,18 +453,16 @@ class DataLoggerProgramFiles(Base): class DataLoggerFiles(Base): - DataLoggerFileID = Column('dataloggerfileid', Integer, primary_key=True, nullable=False) - ProgramID = Column('actionid', Integer, ForeignKey(DataLoggerProgramFiles.ProgramID), nullable=False) + ProgramID = Column('programid', Integer, ForeignKey(DataLoggerProgramFiles.ProgramID), nullable=False) DataLoggerFileName = Column('dataloggerfilename', String(255), nullable=False) - DataLoggerOutputFileDescription = Column('dataloggeroutputfiledescription', String(500)) - DataLoggerOutputFileLink = Column('dataloggeroutputfilelink', String(255)) + DataLoggerOutputFileDescription = Column('dataloggerfiledescription', String(500)) + DataLoggerOutputFileLink = Column('dataloggerfilelink', String(255)) ProgramObj = relationship(DataLoggerProgramFiles) class EquipmentModels(Base): - ModelID = Column('modelid', Integer, primary_key=True, nullable=False) ModelManufacturerID = Column('modelmanufacturerid', Integer, ForeignKey(Organizations.OrganizationID), nullable=False) @@ -479,7 +477,6 @@ class EquipmentModels(Base): class InstrumentOutputVariables(Base): - InstrumentOutputVariableID = Column( 'instrumentoutputvariableid', Integer, @@ -501,9 +498,8 @@ class InstrumentOutputVariables(Base): class DataLoggerFileColumns(Base): - DataLoggerFileColumnID = Column('dataloggerfilecolumnid', Integer, primary_key=True, nullable=False) - ResultID = Column('resultid', BigInteger, ForeignKey(Results.ResultID)) + ResultID = Column('resultid', BigIntegerType, ForeignKey(Results.ResultID)) DataLoggerFileID = Column('dataloggerfileid', Integer, ForeignKey(DataLoggerFiles.DataLoggerFileID), nullable=False) InstrumentOutputVariableID = Column('instrumentoutputvariableid', Integer, @@ -511,7 +507,7 @@ class DataLoggerFileColumns(Base): nullable=False) ColumnLabel = Column('columnlabel', String(50), nullable=False) ColumnDescription = Column('columndescription', String(500)) - MeasurementEquation = Column('measurmentequation', String(255)) + MeasurementEquation = Column('measurementequation', String(255)) ScanInterval = Column('scaninterval', Float(50)) ScanIntervalUnitsID = Column('scanintervalunitsid', Integer, ForeignKey(Units.UnitsID)) RecordingInterval = Column('recordinginterval', Float(50)) @@ -537,7 +533,6 @@ class DataLoggerFileColumns(Base): class Equipment(Base): - EquipmentID = Column('equipmentid', Integer, primary_key=True, nullable=False) EquipmentCode = Column('equipmentcode', String(50), nullable=False) EquipmentName = Column('equipmentname', String(255), nullable=False) @@ -557,7 +552,6 @@ class Equipment(Base): class CalibrationReferenceEquipment(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ActionID = Column('actionid', Integer, ForeignKey(Actions.ActionID), nullable=False) EquipmentID = Column('equipmentid', Integer, ForeignKey(Equipment.EquipmentID), nullable=False) @@ -567,7 +561,6 @@ class CalibrationReferenceEquipment(Base): class EquipmentActions(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) EquipmentID = Column('equipmentid', ForeignKey(Equipment.EquipmentID), nullable=False) ActionID = Column('actionid', ForeignKey(Actions.ActionID), nullable=False) @@ -577,7 +570,6 @@ class EquipmentActions(Base): class EquipmentUsed(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ActionID = Column('actionid', Integer, ForeignKey(Actions.ActionID), nullable=False) EquipmentID = Column('equipmentid', Integer, ForeignKey(Equipment.EquipmentID), nullable=False) @@ -587,8 +579,7 @@ class EquipmentUsed(Base): class MaintenanceActions(Base): - - ActionID = Column('actionid', Integer, ForeignKey(Actions.ActionID), primary_key=True, nullable=False) + ActionID = Column('actionid', Integer, ForeignKey(Actions.ActionID), primary_key=True, nullable=False) IsFactoryService = Column('isfactoryservce', Boolean, nullable=False) MaintenanceCode = Column('maintenancecode', String(50)) MantenanceReason = Column('maintenancereason', String(50)) @@ -597,7 +588,6 @@ class MaintenanceActions(Base): class RelatedEquipment(Base): - RelationID = Column('relationid', Integer, primary_key=True, nullable=True) EquipmentID = Column('equipmentid', Integer, ForeignKey(Equipment.EquipmentID), nullable=True) RelationshipTypeCV = Column('relationshiptypecv', String(255), nullable=True, index=True) @@ -623,8 +613,7 @@ class RelatedEquipment(Base): class CalibrationActions(Base): - - ActionID = Column('actionid', Integer, ForeignKey(Actions.ActionID), primary_key=True, nullable=False) + ActionID = Column('actionid', Integer, ForeignKey(Actions.ActionID), primary_key=True, nullable=False) CalibrationCheckValue = Column('calibrationcheckvalue', Float(53)) InstrumentOutputVariableID = Column('instrumentoutputvariableid', Integer, ForeignKey(InstrumentOutputVariables.VariableID), nullable=False) @@ -633,20 +622,19 @@ class CalibrationActions(Base): ActionObj = relationship(Actions) InstrumentOutputVariableObj = relationship(InstrumentOutputVariables) + # ################################################################################ # Lab Analyses # ################################################################################ class Directives(Base): - DirectiveID = Column('directiveid', Integer, primary_key=True, nullable=False) DirectiveTypeCV = Column('directivetypecv', ForeignKey(CVDirectiveType.Name), nullable=False, index=True) DirectiveDescription = Column('directivedescription', String(500), nullable=False) class ActionDirectives(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ActionID = Column('actionid', ForeignKey(Actions.ActionID), nullable=False) DirectiveID = Column('directiveid', ForeignKey(Directives.DirectiveID), nullable=False) @@ -676,7 +664,6 @@ class SpecimenBatchPositions(Base): # Sampling Features # ################################################################################ class SpatialReferences(Base): - SpatialReferenceID = Column('spatialreferenceid', Integer, primary_key=True, nullable=False) SRSCode = Column('srscode', String(50)) SRSName = Column('srsname', String(255), nullable=False) @@ -685,7 +672,6 @@ class SpatialReferences(Base): class Specimens(SamplingFeatures): - SamplingFeatureID = Column('samplingfeatureid', ForeignKey(SamplingFeatures.SamplingFeatureID), primary_key=True) SpecimenTypeCV = Column('specimentypecv', ForeignKey(CVSpecimenType.Name), nullable=False, index=True) @@ -698,7 +684,6 @@ class Specimens(SamplingFeatures): class SpatialOffsets(Base): - SpatialOffsetID = Column('spatialoffsetid', Integer, primary_key=True, nullable=False) SpatialOffsetTypeCV = Column('spatialoffsettypecv', ForeignKey(CVSpatialOffsetType.Name), nullable=False, index=True) @@ -715,7 +700,6 @@ class SpatialOffsets(Base): class Sites(SamplingFeatures): - SamplingFeatureID = Column('samplingfeatureid', ForeignKey(SamplingFeatures.SamplingFeatureID), primary_key=True) SpatialReferenceID = Column('spatialreferenceid', ForeignKey(SpatialReferences.SpatialReferenceID), @@ -732,7 +716,6 @@ class Sites(SamplingFeatures): class RelatedFeatures(Base): - RelationID = Column('relationid', Integer, primary_key=True, nullable=False) SamplingFeatureID = Column('samplingfeatureid', ForeignKey(SamplingFeatures.SamplingFeatureID), nullable=False) @@ -757,7 +740,6 @@ class RelatedFeatures(Base): class SpecimenTaxonomicClassifiers(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) SamplingFeatureID = Column('samplingfeatureid', ForeignKey(Specimens.SamplingFeatureID), nullable=False) TaxonomicClassifierID = Column('taxonomicclassifierid', @@ -772,7 +754,6 @@ class SpecimenTaxonomicClassifiers(Base): # Simulation # ################################################################################ class Models(Base): - ModelID = Column('modelid', Integer, primary_key=True, nullable=False) ModelCode = Column('modelcode', String(255), nullable=False) ModelName = Column('modelname', String(255), nullable=False) @@ -780,7 +761,6 @@ class Models(Base): class RelatedModels(Base): - RelatedID = Column('relatedid', Integer, primary_key=True, nullable=False) ModelID = Column('modelid', ForeignKey(Models.ModelID), nullable=False) RelationshipTypeCV = Column('relationshiptypecv', ForeignKey(CVRelationshipType.Name), nullable=False, @@ -792,7 +772,6 @@ class RelatedModels(Base): class Simulations(Base): - SimulationID = Column('simulationid', Integer, primary_key=True, nullable=False) ActionID = Column('actionid', ForeignKey(Actions.ActionID), nullable=False) SimulationName = Column('simulationname', String(255), nullable=False) @@ -815,7 +794,6 @@ class Simulations(Base): # Part of the Provenance table, needed here to meet dependencies class Citations(Base): - CitationID = Column('citationid', Integer, primary_key=True, nullable=False) Title = Column('title', String(255), nullable=False) Publisher = Column('publisher', String(255), nullable=False) @@ -827,7 +805,6 @@ class Citations(Base): # Annotations # ################################################################################ class Annotations(Base): - AnnotationID = Column('annotationid', Integer, primary_key=True, nullable=False) AnnotationTypeCV = Column( 'annotationtypecv', @@ -849,7 +826,6 @@ class Annotations(Base): class ActionAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ActionID = Column('actionid', ForeignKey(Actions.ActionID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) @@ -859,9 +835,8 @@ class ActionAnnotations(Base): class EquipmentAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - EquipmentID = Column('valueid', BigInteger, ForeignKey(Equipment.EquipmentID), nullable=False) + EquipmentID = Column('valueid', BigIntegerType, ForeignKey(Equipment.EquipmentID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) @@ -869,7 +844,6 @@ class EquipmentAnnotations(Base): class MethodAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) MethodID = Column('methodid', ForeignKey(Methods.MethodID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) @@ -879,7 +853,6 @@ class MethodAnnotations(Base): class ResultAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ResultID = Column('resultid', ForeignKey(Results.ResultID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) @@ -891,7 +864,6 @@ class ResultAnnotations(Base): class SamplingFeatureAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) SamplingFeatureID = Column('samplingfeatureid', ForeignKey(SamplingFeatures.SamplingFeatureID), nullable=False) @@ -905,7 +877,6 @@ class SamplingFeatureAnnotations(Base): # Data Quality # ################################################################################ class DataSetsResults(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) DataSetID = Column('datasetid', ForeignKey(DataSets.DataSetID), nullable=False) ResultID = Column('resultid', ForeignKey(Results.ResultID), nullable=False) @@ -915,7 +886,6 @@ class DataSetsResults(Base): class DataQuality(Base): - DataQualityID = Column('dataqualityid', Integer, primary_key=True, nullable=False) DataQualityTypeCV = Column('dataqualitytypecv', ForeignKey(CVDataQualityType.Name), nullable=False, index=True) @@ -929,7 +899,6 @@ class DataQuality(Base): class ReferenceMaterials(Base): - ReferenceMaterialID = Column('referencematerialid', Integer, primary_key=True, nullable=False) ReferenceMaterialMediumCV = Column( 'referencematerialmediumcv', @@ -951,7 +920,6 @@ class ReferenceMaterials(Base): class CalibrationStandards(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ActionID = Column('actionid', Integer, ForeignKey(Actions.ActionID), nullable=False) ReferenceMaterialID = Column( @@ -966,7 +934,6 @@ class CalibrationStandards(Base): class ReferenceMaterialValues(Base): - ReferenceMaterialValueID = Column('referencematerialvalueid', Integer, primary_key=True, nullable=False) ReferenceMaterialID = Column('referencematerialid', ForeignKey(ReferenceMaterials.ReferenceMaterialID), nullable=False) @@ -983,7 +950,6 @@ class ReferenceMaterialValues(Base): class ResultNormalizationValues(Base): - ResultID = Column(u'resultid', ForeignKey(Results.ResultID), primary_key=True) ReferenceMaterialValueID = Column(u'referencematerialvalueid', ForeignKey(ReferenceMaterialValues.ReferenceMaterialValueID), @@ -994,7 +960,6 @@ class ResultNormalizationValues(Base): class ResultsDataQuality(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ResultID = Column('resultid', ForeignKey(Results.ResultID), nullable=False) DataQualityID = Column('dataqualityid', ForeignKey(DataQuality.DataQualityID), nullable=False) @@ -1007,7 +972,6 @@ class ResultsDataQuality(Base): # Extension Properties # ################################################################################ class ExtensionProperties(Base): - PropertyID = Column('propertyid', Integer, primary_key=True, nullable=False) PropertyName = Column('propertyname', String(255), nullable=False) PropertyDescription = Column('propertydescription', String(500)) @@ -1019,7 +983,6 @@ class ExtensionProperties(Base): class ActionExtensionPropertyValues(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ActionID = Column('actionid', ForeignKey(Actions.ActionID), nullable=False) PropertyID = Column('propertyid', ForeignKey(ExtensionProperties.PropertyID), nullable=False) @@ -1030,7 +993,6 @@ class ActionExtensionPropertyValues(Base): class CitationExtensionPropertyValues(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) CitationID = Column('citationid', ForeignKey(Citations.CitationID), nullable=False) PropertyID = Column('propertyid', ForeignKey(ExtensionProperties.PropertyID), nullable=False) @@ -1041,7 +1003,6 @@ class CitationExtensionPropertyValues(Base): class MethodExtensionPropertyValues(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) MethodID = Column('methodid', ForeignKey(Methods.MethodID), nullable=False) PropertyID = Column('propertyid', ForeignKey(ExtensionProperties.PropertyID), nullable=False) @@ -1052,7 +1013,6 @@ class MethodExtensionPropertyValues(Base): class ResultExtensionPropertyValues(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ResultID = Column('resultid', ForeignKey(Results.ResultID), nullable=False) PropertyID = Column('propertyid', ForeignKey(ExtensionProperties.PropertyID), nullable=False) @@ -1063,7 +1023,6 @@ class ResultExtensionPropertyValues(Base): class SamplingFeatureExtensionPropertyValues(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) SamplingFeatureID = Column('samplingfeatureid', ForeignKey(SamplingFeatures.SamplingFeatureID), nullable=False) @@ -1075,7 +1034,6 @@ class SamplingFeatureExtensionPropertyValues(Base): class VariableExtensionPropertyValues(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) VariableID = Column('variableid', ForeignKey(Variables.VariableID), nullable=False) PropertyID = Column('propertyid', ForeignKey(ExtensionProperties.PropertyID), nullable=False) @@ -1089,7 +1047,6 @@ class VariableExtensionPropertyValues(Base): # Extension Identifiers # ################################################################################ class ExternalIdentifierSystems(Base): - ExternalIdentifierSystemID = Column( 'externalidentifiersystemid', Integer, @@ -1106,7 +1063,6 @@ class ExternalIdentifierSystems(Base): class CitationExternalIdentifiers(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) CitationID = Column('citationid', ForeignKey(Citations.CitationID), nullable=False) ExternalIdentifierSystemID = Column('externalidentifiersystemid', @@ -1120,7 +1076,6 @@ class CitationExternalIdentifiers(Base): class MethodExternalIdentifiers(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) MethodID = Column('methodid', ForeignKey(Methods.MethodID), nullable=False) ExternalIdentifierSystemID = Column('externalidentifiersystemid', @@ -1135,7 +1090,6 @@ class MethodExternalIdentifiers(Base): class PersonExternalIdentifiers(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) PersonID = Column('personid', ForeignKey(People.PersonID), nullable=False) ExternalIdentifierSystemID = Column('externalidentifiersystemid', @@ -1149,7 +1103,6 @@ class PersonExternalIdentifiers(Base): class ReferenceMaterialExternalIdentifiers(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) ReferenceMaterialID = Column(ForeignKey(ReferenceMaterials.ReferenceMaterialID), nullable=False) ExternalIdentifierSystemID = Column('externalidentifiersystemid', @@ -1167,7 +1120,6 @@ class ReferenceMaterialExternalIdentifiers(Base): class SamplingFeatureExternalIdentifiers(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) SamplingFeatureID = Column('samplingfeatureid', ForeignKey(SamplingFeatures.SamplingFeatureID), nullable=False) @@ -1186,7 +1138,6 @@ class SamplingFeatureExternalIdentifiers(Base): class SpatialReferenceExternalIdentifiers(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) SpatialReferenceID = Column('spatialreferenceid', ForeignKey(SpatialReferences.SpatialReferenceID), nullable=False) @@ -1205,7 +1156,6 @@ class SpatialReferenceExternalIdentifiers(Base): class TaxonomicClassifierExternalIdentifiers(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) TaxonomicClassifierID = Column('taxonomicclassifierid', ForeignKey(TaxonomicClassifiers.TaxonomicClassifierID), nullable=False) @@ -1224,7 +1174,6 @@ class TaxonomicClassifierExternalIdentifiers(Base): class VariableExternalIdentifiers(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) VariableID = Column('variableid', ForeignKey(Variables.VariableID), nullable=False) ExternalIdentifierSystemID = Column('externalidentifiersystemid', @@ -1242,7 +1191,6 @@ class VariableExternalIdentifiers(Base): # ################################################################################ class AuthorLists(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) CitationID = Column('citationid', ForeignKey(Citations.CitationID), nullable=False) PersonID = Column('personid', ForeignKey(People.PersonID), nullable=False) @@ -1253,7 +1201,6 @@ class AuthorLists(Base): class DataSetCitations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) DataSetID = Column('datasetid', ForeignKey(DataSets.DataSetID), nullable=False) RelationshipTypeCV = Column('relationshiptypecv', ForeignKey(CVRelationshipType.Name), nullable=False, @@ -1265,13 +1212,11 @@ class DataSetCitations(Base): class DerivationEquations(Base): - DerivationEquationID = Column('derivationequationid', Integer, primary_key=True, nullable=False) DerivationEquation = Column('derivationequation', String(255), nullable=False) class ResultDerivationEquations(Base): - ResultID = Column(u'resultid', ForeignKey(Results.ResultID), primary_key=True) DerivationEquationID = Column( u'derivationequationid', @@ -1284,7 +1229,6 @@ class ResultDerivationEquations(Base): class MethodCitations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) MethodID = Column('methodid', ForeignKey(Methods.MethodID), nullable=False) RelationshipTypeCV = Column('relationshiptypecv', ForeignKey(CVRelationshipType.Name), nullable=False, @@ -1296,7 +1240,6 @@ class MethodCitations(Base): class RelatedAnnotations(Base): - RelationID = Column('relationid', Integer, primary_key=True, nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) RelationshipTypeCV = Column('relationshiptypecv', ForeignKey(CVRelationshipType.Name), nullable=False, @@ -1314,7 +1257,6 @@ class RelatedAnnotations(Base): class RelatedCitations(Base): - RelationID = Column('relationid', Integer, primary_key=True, nullable=False) CitationID = Column('citationid', ForeignKey(Citations.CitationID), nullable=False) RelationshipTypeCV = Column('relationshiptypecv', ForeignKey(CVRelationshipType.Name), nullable=False, @@ -1329,7 +1271,6 @@ class RelatedCitations(Base): class RelatedDataSets(Base): - RelationID = Column('relationid', Integer, primary_key=True, nullable=False) DataSetID = Column('datasetid', ForeignKey(DataSets.DataSetID), nullable=False) RelationshipTypeCV = Column('relationshiptypecv', ForeignKey(CVRelationshipType.Name), nullable=False, @@ -1345,7 +1286,6 @@ class RelatedDataSets(Base): class RelatedResults(Base): - RelationID = Column('relationid', Integer, primary_key=True, nullable=False) ResultID = Column('resultid', ForeignKey(Results.ResultID), nullable=False) RelationshipTypeCV = Column('relationshiptypecv', ForeignKey(CVRelationshipType.Name), nullable=False, @@ -1362,7 +1302,6 @@ class RelatedResults(Base): # Results # ################################################################################ class PointCoverageResults(Results): - ResultID = Column('resultid', ForeignKey(Results.ResultID), primary_key=True) ZLocation = Column('zlocation', Float(53)) ZLocationUnitsID = Column('zlocationunitsid', ForeignKey(Units.UnitsID)) @@ -1394,7 +1333,6 @@ class PointCoverageResults(Results): class ProfileResults(Results): - ResultID = Column('resultid', ForeignKey(Results.ResultID), primary_key=True) XLocation = Column('xlocation', Float(53)) XLocationUnitsID = Column('xlocationunitsid', ForeignKey(Units.UnitsID)) @@ -1424,7 +1362,6 @@ class ProfileResults(Results): class CategoricalResults(Results): - ResultID = Column('resultid', ForeignKey(Results.ResultID), primary_key=True) XLocation = Column('xlocation', Float(53)) XLocationUnitsID = Column('xlocationunitsid', Integer, ForeignKey(Units.UnitsID)) @@ -1453,7 +1390,6 @@ class CategoricalResults(Results): class TransectResults(Results): - ResultID = Column('resultid', ForeignKey(Results.ResultID), primary_key=True) ZLocation = Column('zlocation', Float(53)) ZLocationUnitsID = Column('zlocationunitsid', ForeignKey(Units.UnitsID)) @@ -1480,7 +1416,6 @@ class TransectResults(Results): class SpectraResults(Results): - ResultID = Column('resultid', ForeignKey(Results.ResultID), primary_key=True) XLocation = Column('xlocation', Float(53)) XLocationUnitsID = Column('xlocationunitsid', ForeignKey(Units.UnitsID)) @@ -1507,7 +1442,6 @@ class SpectraResults(Results): class TimeSeriesResults(Results): - ResultID = Column('resultid', ForeignKey(Results.ResultID), primary_key=True) XLocation = Column('xlocation', Float(53)) XLocationUnitsID = Column('xlocationunitsid', ForeignKey(Units.UnitsID)) @@ -1534,7 +1468,6 @@ class TimeSeriesResults(Results): class SectionResults(Results): - ResultID = Column('resultid', ForeignKey(Results.ResultID), primary_key=True) YLocation = Column('ylocation', Float(53)) YLocationUnitsID = Column('ylocationunitsid', ForeignKey(Units.UnitsID)) @@ -1573,7 +1506,6 @@ class SectionResults(Results): class TrajectoryResults(Results): - ResultID = Column('resultid', ForeignKey(Results.ResultID), primary_key=True) SpatialReferenceID = Column('spatialreferenceid', ForeignKey(SpatialReferences.SpatialReferenceID)) IntendedTrajectorySpacing = Column('intendedtrajectoryspacing', Float(53)) @@ -1597,7 +1529,6 @@ class TrajectoryResults(Results): class MeasurementResults(Results): - ResultID = Column('resultid', ForeignKey(Results.ResultID), primary_key=True) XLocation = Column('xlocation', Float(53)) XLocationUnitsID = Column('xlocationunitsid', ForeignKey(Units.UnitsID)) @@ -1636,33 +1567,30 @@ class MeasurementResults(Results): class CategoricalResultValues(Base): - ValueID = Column('valueid', BigIntegerType, primary_key=True) ResultID = Column('resultid', ForeignKey(CategoricalResults.ResultID), nullable=False) DataValue = Column('datavalue', String(255), nullable=False) - ValueDateTime = Column('valuedatetime', DateTime, nullable=False) + ValueDateTime = Column('valuedatetime', DateTimeType, nullable=False) ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', Integer, nullable=False) ResultObj = relationship(CategoricalResults) class MeasurementResultValues(Base): - ValueID = Column('valueid', BigIntegerType, primary_key=True) ResultID = Column('resultid', ForeignKey(MeasurementResults.ResultID), nullable=False) DataValue = Column('datavalue', Float(53), nullable=False) - ValueDateTime = Column('valuedatetime', DateTime, nullable=False) + ValueDateTime = Column('valuedatetime', DateTimeType, nullable=False) ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', Integer, nullable=False) ResultObj = relationship(MeasurementResults) class PointCoverageResultValues(Base): - ValueID = Column('valueid', BigIntegerType, primary_key=True) ResultID = Column('resultid', ForeignKey(PointCoverageResults.ResultID), nullable=False) - DataValue = Column('datavalue', BigInteger, nullable=False) - ValueDateTime = Column('valuedatetime', DateTime, nullable=False) + DataValue = Column('datavalue', BigIntegerType, nullable=False) + ValueDateTime = Column('valuedatetime', DateTimeType, nullable=False) ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', Integer, nullable=False) XLocation = Column('xlocation', Float(53), nullable=False) XLocationUnitsID = Column('xlocationunitsid', ForeignKey(Units.UnitsID), nullable=False) @@ -1683,11 +1611,10 @@ class PointCoverageResultValues(Base): class ProfileResultValues(Base): - ValueID = Column('valueid', BigIntegerType, primary_key=True) ResultID = Column('resultid', ForeignKey(ProfileResults.ResultID), nullable=False) DataValue = Column('datavalue', Float(53), nullable=False) - ValueDateTime = Column('valuedatetime', DateTime, nullable=False) + ValueDateTime = Column('valuedatetime', DateTimeType, nullable=False) ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', Integer, nullable=False) ZLocation = Column('zlocation', Float(53), nullable=False) ZAggregationInterval = Column('zaggregationinterval', Float(53), nullable=False) @@ -1710,16 +1637,15 @@ class ProfileResultValues(Base): class SectionResultValues(Base): - ValueID = Column('valueid', BigIntegerType, primary_key=True) ResultID = Column('resultid', ForeignKey(SectionResults.ResultID), nullable=False) DataValue = Column('datavalue', Float(53), nullable=False) - ValueDateTime = Column('valuedatetime', BigInteger, nullable=False) - ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', BigInteger, nullable=False) + ValueDateTime = Column('valuedatetime', DateTimeType, nullable=False) + ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', Integer, nullable=False) XLocation = Column('xlocation', Float(53), nullable=False) XAggregationInterval = Column('xaggregationinterval', Float(53), nullable=False) XLocationUnitsID = Column('xlocationunitsid', ForeignKey(Units.UnitsID), nullable=False) - ZLocation = Column('zlocation', BigInteger, nullable=False) + ZLocation = Column('zlocation', BigIntegerType, nullable=False) ZAggregationInterval = Column('zaggregationinterval', Float(53), nullable=False) ZLocationUnitsID = Column('zlocationunitsid', ForeignKey(Units.UnitsID), nullable=False) CensorCodeCV = Column('censorcodecv', ForeignKey(CVCensorCode.Name), nullable=False, index=True) @@ -1746,11 +1672,10 @@ class SectionResultValues(Base): class SpectraResultValues(Base): - ValueID = Column('valueid', BigIntegerType, primary_key=True) ResultID = Column('resultid', ForeignKey(SpectraResults.ResultID), nullable=False) DataValue = Column('datavalue', Float(53), nullable=False) - ValueDateTime = Column('valuedatetime', DateTime, nullable=False) + ValueDateTime = Column('valuedatetime', DateTimeType, nullable=False) ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', Integer, nullable=False) ExcitationWavelength = Column('excitationwavelength', Float(53), nullable=False) EmissionWavelength = Column('emmistionwavelength', Float(53), nullable=False) @@ -1773,13 +1698,11 @@ class SpectraResultValues(Base): ) - class TimeSeriesResultValues(Base): - ValueID = Column('valueid', BigIntegerType, primary_key=True) ResultID = Column('resultid', ForeignKey(TimeSeriesResults.ResultID), nullable=False) DataValue = Column('datavalue', Float(53), nullable=False) - ValueDateTime = Column('valuedatetime', DateTime, nullable=False) + ValueDateTime = Column('valuedatetime', DateTimeType, nullable=False) ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', Integer, nullable=False) CensorCodeCV = Column('censorcodecv', ForeignKey(CVCensorCode.Name), nullable=False, index=True) QualityCodeCV = Column('qualitycodecv', ForeignKey(CVQualityCode.Name), nullable=False, index=True) @@ -1801,11 +1724,10 @@ def list_repr(self): class TrajectoryResultValues(Base): - ValueID = Column('valueid', BigIntegerType, primary_key=True) ResultID = Column('resultid', ForeignKey(TrajectoryResults.ResultID), nullable=False) DataValue = Column('datavalue', Float(53), nullable=False) - ValueDateTime = Column('valuedatetime', DateTime, nullable=False) + ValueDateTime = Column('valuedatetime', DateTimeType, nullable=False) ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', Integer, nullable=False) XLocation = Column('xlocation', Float(53), nullable=False) XLocationUnitsID = Column('xlocationunitsid', ForeignKey(Units.UnitsID), nullable=False) @@ -1846,12 +1768,11 @@ class TrajectoryResultValues(Base): class TransectResultValues(Base): - ValueID = Column('valueid', BigIntegerType, primary_key=True) ResultID = Column('resultid', ForeignKey(TransectResults.ResultID), nullable=False) DataValue = Column('datavalue', Float(53), nullable=False) - ValueDateTime = Column('valuedatetime', DateTime, nullable=False) - ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', DateTime, nullable=False) + ValueDateTime = Column('valuedatetime', DateTimeType, nullable=False) + ValueDateTimeUTCOffset = Column('valuedatetimeutcoffset', Integer, nullable=False) XLocation = Column('xlocation', Float(53), nullable=False) XLocationUnitsID = Column('xlocationunitsid', ForeignKey(Units.UnitsID), nullable=False) YLocation = Column('ylocation', Float(53), nullable=False) @@ -1894,9 +1815,8 @@ class TransectResultValues(Base): class CategoricalResultValueAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - ValueID = Column('valueid', BigInteger, ForeignKey(CategoricalResultValues.ValueID), nullable=False) + ValueID = Column('valueid', BigIntegerType, ForeignKey(CategoricalResultValues.ValueID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) @@ -1904,9 +1824,8 @@ class CategoricalResultValueAnnotations(Base): class MeasurementResultValueAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - ValueID = Column('valueid', BigInteger, ForeignKey(MeasurementResultValues.ValueID), nullable=False) + ValueID = Column('valueid', BigIntegerType, ForeignKey(MeasurementResultValues.ValueID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) @@ -1914,9 +1833,8 @@ class MeasurementResultValueAnnotations(Base): class PointCoverageResultValueAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - ValueID = Column('valueid', BigInteger, ForeignKey(PointCoverageResultValues.ValueID), nullable=False) + ValueID = Column('valueid', BigIntegerType, ForeignKey(PointCoverageResultValues.ValueID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) @@ -1924,9 +1842,8 @@ class PointCoverageResultValueAnnotations(Base): class ProfileResultValueAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - ValueID = Column('valueid', BigInteger, ForeignKey(ProfileResultValues.ValueID), nullable=False) + ValueID = Column('valueid', BigIntegerType, ForeignKey(ProfileResultValues.ValueID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) @@ -1934,9 +1851,8 @@ class ProfileResultValueAnnotations(Base): class SectionResultValueAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - ValueID = Column('valueid', BigInteger, ForeignKey(SectionResultValues.ValueID), nullable=False) + ValueID = Column('valueid', BigIntegerType, ForeignKey(SectionResultValues.ValueID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) @@ -1944,9 +1860,8 @@ class SectionResultValueAnnotations(Base): class SpectraResultValueAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - ValueID = Column('valueid', BigInteger, ForeignKey(SpectraResultValues.ValueID), nullable=False) + ValueID = Column('valueid', BigIntegerType, ForeignKey(SpectraResultValues.ValueID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) @@ -1954,9 +1869,8 @@ class SpectraResultValueAnnotations(Base): class TimeSeriesResultValueAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - ValueID = Column('valueid', BigInteger, ForeignKey(TimeSeriesResultValues.ValueID), nullable=False) + ValueID = Column('valueid', BigIntegerType, ForeignKey(TimeSeriesResultValues.ValueID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) @@ -1964,9 +1878,8 @@ class TimeSeriesResultValueAnnotations(Base): class TrajectoryResultValueAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - ValueID = Column('valueid', BigInteger, ForeignKey(TrajectoryResultValues.ValueID), nullable=False) + ValueID = Column('valueid', BigIntegerType, ForeignKey(TrajectoryResultValues.ValueID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) @@ -1974,9 +1887,8 @@ class TrajectoryResultValueAnnotations(Base): class TransectResultValueAnnotations(Base): - BridgeID = Column('bridgeid', Integer, primary_key=True, nullable=False) - ValueID = Column('valueid', BigInteger, ForeignKey(TransectResultValues.ValueID), nullable=False) + ValueID = Column('valueid', BigIntegerType, ForeignKey(TransectResultValues.ValueID), nullable=False) AnnotationID = Column('annotationid', ForeignKey(Annotations.AnnotationID), nullable=False) AnnotationObj = relationship(Annotations) diff --git a/odm2api/ODM2/services/readService.py b/odm2api/ODM2/services/readService.py index 5816127..cbe8cc4 100644 --- a/odm2api/ODM2/services/readService.py +++ b/odm2api/ODM2/services/readService.py @@ -1,5 +1,7 @@ from __future__ import (absolute_import, division, print_function) +import warnings + from odm2api.ODM2 import serviceBase from odm2api.ODM2.models import ( ActionAnnotations, ActionDirectives, ActionExtensionPropertyValues, Actions, @@ -12,7 +14,7 @@ CalibrationActions, CalibrationReferenceEquipment, CalibrationStandards, CategoricalResultValueAnnotations, CategoricalResultValues, CitationExtensionPropertyValues, CitationExternalIdentifiers, DataLoggerFileColumns, DataLoggerFiles, DataLoggerProgramFiles, - DataQuality, DataSetCitations, DataSets, DerivationEquations, Directives, Equipment, + DataQuality, DataSetCitations, DataSets, DataSetsResults, DerivationEquations, Directives, Equipment, EquipmentActions, EquipmentAnnotations, EquipmentModels, EquipmentUsed, ExtensionProperties, ExternalIdentifierSystems, FeatureActions, InstrumentOutputVariables, MaintenanceActions, MeasurementResultValueAnnotations, MeasurementResultValues, MethodAnnotations, @@ -29,7 +31,7 @@ SpatialReferenceExternalIdentifiers, SpatialReferences, SpecimenBatchPositions, SpectraResultValueAnnotations, SpectraResultValues, TaxonomicClassifierExternalIdentifiers, TaxonomicClassifiers, TimeSeriesResultValueAnnotations, TimeSeriesResultValues, - TimeSeriesResults, TrajectoryResultValueAnnotations, TrajectoryResultValues, + TrajectoryResultValueAnnotations, TrajectoryResultValues, TransectResultValueAnnotations, TransectResultValues, Units, VariableExtensionPropertyValues, VariableExternalIdentifiers, Variables, ) @@ -72,7 +74,83 @@ def __init__(self, affiliation, person, org): self.Organization = '(' + org.OrganizationCode + ') ' + org.OrganizationName +class SamplingFeatureDataSet(): + datasets = {} + related_features = {} + + def __init__(self, samplingfeature, datasetresults, relatedfeatures): + sf = samplingfeature + + self.SamplingFeature = sf + self.SamplingFeatureID = sf.SamplingFeatureID + self.SamplingFeatureUUID = sf.SamplingFeatureUUID + self.SamplingFeatureTypeCV = sf.SamplingFeatureTypeCV + self.SamplingFeatureCode = sf.SamplingFeatureCode + self.SamplingFeatureName = sf.SamplingFeatureName + self.SamplingFeatureDescription = sf.SamplingFeatureDescription + self.SamplingFeatureGeotypeCV = sf.SamplingFeatureGeotypeCV + self.Elevation_m = sf.Elevation_m + self.ElevationDatumCV = sf.ElevationDatumCV + self.FeatureGeometryWKT = sf.FeatureGeometryWKT + self.assignDatasets(datasetresults) + self.assignRelatedFeatures(relatedfeatures) + + print(self.datasets) + + def assignDatasets(self, datasetresults): + self.datasets = {} + if datasetresults: + for dsr in datasetresults: + if dsr.DataSetObj not in self.datasets: + # if the dataset is not in the dictionary, add it and the first result + self.datasets[dsr.DataSetObj] = [] + res = dsr.ResultObj + # res.FeatureActionObj = None + self.datasets[dsr.DataSetObj].append(res) + else: + # if the dataset is in the dictionary, append the result object to the list + res = dsr.ResultObj + # res.FeatureActionObj = None + self.datasets[dsr.DataSetObj].append(res) + + def assignRelatedFeatures(self, relatedfeatures): + self.related_features = {} + if relatedfeatures: + for related in relatedfeatures: + if related.SamplingFeatureTypeCV == 'Site': + self.related_features = related + + class ReadODM2(serviceBase): + def _get_columns(self, model): + """Internal helper function to get a dictionary of a model column properties. + + Args: + model (object): Sqlalchemy object, Ex. ODM2 model. + + Returns: + dict: Dictionary of column properties Ex. {'resultid': 'ResultID'} + + """ + from sqlalchemy.orm.properties import ColumnProperty + columns = [(prop.key.lower(), prop.key) for prop in model.__mapper__.iterate_properties if + isinstance(prop, ColumnProperty)] + + return dict(columns) + + def _check_kwargs(self, args, kwargs): + """Internal helper function to check for unused keyword arguments + + Args: + args (list): List of expected, valid arguments. + kwargs (dict): Dictionary of keyword arguments from user + Returns: + None + """ + invkwd = filter(lambda x: x not in args, kwargs.keys()) + if invkwd: + warnings.warn('Got unexpected keyword argument(s) {}'.format(','.join(invkwd)), stacklevel=2) + # Exists functions def resultExists(self, result): """ @@ -90,11 +168,12 @@ def resultExists(self, result): ) return ret.scalar() - except: + except Exception as e: + print('Error running Query: {}'.format(e)) return None # Annotations - def getAnnotations(self, type=None, codes=None, ids=None): + def getAnnotations(self, annottype=None, codes=None, ids=None, **kwargs): """ * Pass Nothing - return a list of all objects * Pass AnnotationTypeCV - return a list of all objects of the fiven type @@ -104,34 +183,39 @@ def getAnnotations(self, type=None, codes=None, ids=None): """ # TODO What keywords do I use for type. a = Annotations - if type: - if type == 'action': + self._check_kwargs(['type'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the annottype parameter instead.', + DeprecationWarning, stacklevel=2) + annottype = kwargs['type'] + if annottype: + if annottype == 'action': a = ActionAnnotations - elif type == 'categoricalresultvalue': + elif annottype == 'categoricalresultvalue': a = CategoricalResultValueAnnotations - elif type == 'equipmentannotation': + elif annottype == 'equipmentannotation': a = EquipmentAnnotations - elif type == 'measurementresultvalue': + elif annottype == 'measurementresultvalue': a = MeasurementResultValueAnnotations - elif type == 'method': + elif annottype == 'method': a = MethodAnnotations - elif type == 'pointcoverageresultvalue': + elif annottype == 'pointcoverageresultvalue': a = PointCoverageResultValueAnnotations - elif type == 'profileresultvalue': + elif annottype == 'profileresultvalue': a = ProfileResultValueAnnotations - elif type == 'result': + elif annottype == 'result': a = ResultAnnotations - elif type == 'samplingfeature': + elif annottype == 'samplingfeature': a = SamplingFeatureAnnotations - elif type == 'sectionresultvalue': + elif annottype == 'sectionresultvalue': a = SectionResultValueAnnotations - elif type == 'spectraresultvalue': + elif annottype == 'spectraresultvalue': a = SpectraResultValueAnnotations - elif type == 'timeseriesresultvalue': + elif annottype == 'timeseriesresultvalue': a = TimeSeriesResultValueAnnotations - elif type == 'trajectoryresultvalue': + elif annottype == 'trajectoryresultvalue': a = TrajectoryResultValueAnnotations - elif type == 'transectresultvalue': + elif annottype == 'transectresultvalue': a = TransectResultValueAnnotations try: query = self._session.query(a) @@ -141,70 +225,76 @@ def getAnnotations(self, type=None, codes=None, ids=None): query = query.filter(Annotations.AnnotationID.in_(ids)) return query.all() - except: + except Exception as e: + print('Error running Query: {}'.format(e)) return None # CV - def getCVs(self, type): + def getCVs(self, cvtype, **kwargs): """ getCVs(self, type): * Pass CVType - return a list of all objects of the given type """ - CV = CVActionType - if type == 'actiontype': + self._check_kwargs(['type'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the cvtype parameter instead.', + DeprecationWarning, stacklevel=2) + cvtype = kwargs['type'] + + if cvtype == 'actiontype': CV = CVActionType - elif type == 'aggregationstatistic': + elif cvtype == 'aggregationstatistic': CV = CVAggregationStatistic - elif type == 'annotationtype': + elif cvtype == 'annotationtype': CV = CVAnnotationType - elif type == 'censorcode': + elif cvtype == 'censorcode': CV = CVCensorCode - elif type == 'dataqualitytype': + elif cvtype == 'dataqualitytype': CV = CVDataQualityType - elif type == 'dataset type': + elif cvtype == 'dataset type': CV = CVDataSetType - elif type == 'Directive Type': + elif cvtype == 'Directive Type': CV = CVDirectiveType - elif type == 'Elevation Datum': + elif cvtype == 'Elevation Datum': CV = CVElevationDatum - elif type == 'Equipment Type': + elif cvtype == 'Equipment Type': CV = CVEquipmentType - elif type == 'Medium': + elif cvtype == 'Medium': CV = CVMediumType - elif type == 'Method Type': + elif cvtype == 'Method Type': CV = CVMethodType - elif type == 'Organization Type': + elif cvtype == 'Organization Type': CV = CVOrganizationType - elif type == 'Property Data Type': + elif cvtype == 'Property Data Type': CV = CVPropertyDataType - elif type == 'Quality Code': + elif cvtype == 'Quality Code': CV = CVQualityCode - elif type == 'Relationship Type': + elif cvtype == 'Relationship Type': CV = CVRelationshipType - elif type == 'Result Type': + elif cvtype == 'Result Type': CV = CVResultType - elif type == 'Sampling Feature Geo-type': + elif cvtype == 'Sampling Feature Geo-type': CV = CVSamplingFeatureGeoType - elif type == 'Sampling Feature Type': + elif cvtype == 'Sampling Feature Type': CV = CVSamplingFeatureType - elif type == 'Site Type': + elif cvtype == 'Site Type': CV = CVSiteType - elif type == 'Spatial Offset Type': + elif cvtype == 'Spatial Offset Type': CV = CVSpatialOffsetType - elif type == 'Speciation': + elif cvtype == 'Speciation': CV = CVSpeciation - elif type == 'Specimen Type': + elif cvtype == 'Specimen Type': CV = CVSpecimenType - elif type == 'Status': + elif cvtype == 'Status': CV = CVStatus - elif type == 'Taxonomic Classifier Type': + elif cvtype == 'Taxonomic Classifier Type': CV = CVTaxonomicClassifierType - elif type == 'Units Type': + elif cvtype == 'Units Type': CV = CVUnitsType - elif type == 'Variable Name': + elif cvtype == 'Variable Name': CV = CVVariableName - elif type == 'Variable Type': + elif cvtype == 'Variable Type': CV = CVVariableType else: return None @@ -295,15 +385,16 @@ def getVariables(self, ids=None, codes=None, sitecode=None, results=False): variables = [ x[0] for x in self._session.query(distinct(Results.VariableID)) - .filter(Results.FeatureActionID == FeatureActions.FeatureActionID) - .filter(FeatureActions.SamplingFeatureID == SamplingFeatures.SamplingFeatureID) - .filter(SamplingFeatures.SamplingFeatureCode == sitecode).all() + .filter(Results.FeatureActionID == FeatureActions.FeatureActionID) + .filter(FeatureActions.SamplingFeatureID == SamplingFeatures.SamplingFeatureID) + .filter(SamplingFeatures.SamplingFeatureCode == sitecode).all() ] if ids: ids = list(set(ids).intersection(variables)) else: ids = variables - except: + except Exception as e: + print('Error running Query: {}'.format(e)) pass if results: @@ -313,7 +404,8 @@ def getVariables(self, ids=None, codes=None, sitecode=None, results=False): ids = list(set(ids).intersection(variables)) else: ids = variables - except: + except Exception as e: + print('Error running Query: {}'.format(e)) pass query = self._session.query(Variables) @@ -328,7 +420,7 @@ def getVariables(self, ids=None, codes=None, sitecode=None, results=False): return None # Method - def getMethods(self, ids=None, codes=None, type=None): + def getMethods(self, ids=None, codes=None, methodtype=None, **kwargs): """ * Pass nothing - returns full list of method objects * Pass a list of MethodIDs - returns a single method object for each given id @@ -336,13 +428,19 @@ def getMethods(self, ids=None, codes=None, type=None): * Pass a MethodType - returns a list of method objects of the given MethodType """ + self._check_kwargs(['type'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the medtype parameter instead.', + DeprecationWarning, stacklevel=2) + methodtype = kwargs['type'] + q = self._session.query(Methods) if ids: q = q.filter(Methods.MethodID.in_(ids)) if codes: q = q.filter(Methods.MethodCode.in_(codes)) - if type: - q = q.filter_by(MethodTypeCV=type) + if methodtype: + q = q.filter_by(MethodTypeCV=methodtype) try: return q.all() @@ -353,15 +451,28 @@ def getMethods(self, ids=None, codes=None, type=None): # ProcessingLevel def getProcessingLevels(self, ids=None, codes=None): """ - getProcessingLevels(self, ids=None, codes=None) - * Pass nothing - returns full list of ProcessingLevel objects - * Pass a list of ProcessingLevelID - returns a single processingLevel object for each given id - * Pass a list of ProcessingLevelCode - returns a single processingLevel object for each given code + Retrieve a list of Processing Levels + + If no arguments are passed to the function, or their values are None, + all Processing Levels objects in the database will be returned. + + Args: + ids (list, optional): List of Processing Levels IDs. + codes (list, optional): List of Processing Levels Codes. + + + Returns: + list: List of ProcessingLevels Objects + + Examples: + >>> READ = ReadODM2(SESSION_FACTORY) + >>> READ.getProcessingLevels(ids=[1, 3]) + >>> READ.getProcessingLevels(codes=['L1', 'L3']) """ q = self._session.query(ProcessingLevels) if ids: - q = q.filter(ProcessingLevels.ProcessingLevelsID.in_(ids)) + q = q.filter(ProcessingLevels.ProcessingLevelID.in_(ids)) if codes: q = q.filter(ProcessingLevels.ProcessingLevelCode.in_(codes)) @@ -372,7 +483,8 @@ def getProcessingLevels(self, ids=None, codes=None): return None # Sampling Feature - def getSamplingFeatures(self, ids=None, codes=None, uuids=None, type=None, wkt=None, results=False): + def getSamplingFeatures(self, ids=None, codes=None, uuids=None, + sftype=None, wkt=None, results=False, **kwargs): """Retrieve a list of Sampling Feature objects. If no arguments are passed to the function, or their values are None, @@ -382,7 +494,7 @@ def getSamplingFeatures(self, ids=None, codes=None, uuids=None, type=None, wkt=N ids (list, optional): List of SamplingFeatureIDs. codes (list, optional): List of SamplingFeature Codes. uuids (list, optional): List of UUIDs string. - type (str, optional): Type of Sampling Feature from + sftype (str, optional): Type of Sampling Feature from `controlled vocabulary name `_. wkt (str, optional): SamplingFeature Well Known Text. results (bool, optional): Whether or not you want to return only the @@ -403,13 +515,18 @@ def getSamplingFeatures(self, ids=None, codes=None, uuids=None, type=None, wkt=N >>> READ.getSamplingFeatures(type='Site', results=True) """ + self._check_kwargs(['type'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the sftype parameter instead.', + DeprecationWarning, stacklevel=2) + sftype = kwargs['type'] if results: try: fas = [x[0] for x in self._session.query(distinct(Results.FeatureActionID)).all()] - except: + except Exception as e: + print('Error running Query: {}'.format(e)) return None - sf = [x[0] for x in self._session.query(distinct(FeatureActions.SamplingFeatureID)) - .filter(FeatureActions.FeatureActionID.in_(fas)).all()] + sf = [x[0] for x in self._session.query(distinct(FeatureActions.SamplingFeatureID)).filter(FeatureActions.FeatureActionID.in_(fas)).all()] # noqa if ids: ids = list(set(ids).intersection(sf)) else: @@ -417,8 +534,8 @@ def getSamplingFeatures(self, ids=None, codes=None, uuids=None, type=None, wkt=N q = self._session.query(SamplingFeatures) - if type: - q = q.filter_by(SamplingFeatureTypeCV=type) + if sftype: + q = q.filter_by(SamplingFeatureTypeCV=sftype) if ids: q = q.filter(SamplingFeatures.SamplingFeatureID.in_(ids)) if codes: @@ -443,8 +560,8 @@ def getRelatedSamplingFeatures(self, sfid=None, rfid=None, relationshiptype=None """ - sf = self._session.query(distinct(SamplingFeatures.SamplingFeatureID))\ - .select_from(RelatedFeatures) + sf = self._session.query(distinct(SamplingFeatures.SamplingFeatureID)) \ + .select_from(RelatedFeatures) if sfid: sf = sf.join(RelatedFeatures.RelatedFeatureObj).filter(RelatedFeatures.SamplingFeatureID == sfid) @@ -463,7 +580,7 @@ def getRelatedSamplingFeatures(self, sfid=None, rfid=None, relationshiptype=None return None # Action - def getActions(self, ids=None, type=None, sfid=None): + def getActions(self, ids=None, acttype=None, sfid=None, **kwargs): """ * Pass nothing - returns a list of all Actions * Pass a list of Action ids - returns a list of Action objects @@ -472,12 +589,17 @@ def getActions(self, ids=None, type=None, sfid=None): associated with that Sampling feature ID, Found through featureAction table """ + self._check_kwargs(['type'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the acttype parameter instead.', + DeprecationWarning, stacklevel=2) + acttype = kwargs['type'] a = Actions - if type == 'equipment': + if acttype == 'equipment': a = EquipmentActions - elif type == 'calibration': + elif acttype == 'calibration': a = CalibrationActions - elif type == 'maintenance': + elif acttype == 'maintenance': a = MaintenanceActions q = self._session.query(a) @@ -509,7 +631,7 @@ def getRelatedActions(self, actionid=None): return None # Unit - def getUnits(self, ids=None, name=None, type=None): + def getUnits(self, ids=None, name=None, unittype=None, **kwargs): """ * Pass nothing - returns a list of all units objects * Pass a list of UnitsID - returns a single units object for the given id @@ -517,20 +639,24 @@ def getUnits(self, ids=None, name=None, type=None): * Pass a type- returns a list of all objects of the given type """ + self._check_kwargs(['type'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the unittype parameter instead.', + DeprecationWarning, stacklevel=2) + unittype = kwargs['type'] q = self._session.query(Units) if ids: q = q.filter(Units.UnitsID.in_(ids)) if name: q = q.filter(Units.UnitsName.ilike(name)) - if type: - q = q.filter(Units.UnitsTypeCV.ilike(type)) + if unittype: + q = q.filter(Units.UnitsTypeCV.ilike(unittype)) try: return q.all() except Exception as e: print('Error running Query: {}'.format(e)) return None - # Organization def getOrganizations(self, ids=None, codes=None): """ @@ -612,8 +738,8 @@ def getAffiliations(self, ids=None, personfirst=None, personlast=None, orgcode=N return None # Results - def getResults(self, ids=None, type=None, uuids=None, actionid=None, simulationid=None, sfid=None, - variableid=None, siteid=None): + def getResults(self, ids=None, restype=None, uuids=None, actionid=None, simulationid=None, + variableid=None, siteid=None, sfids=None, sfuuids=None, sfcodes=None, **kwargs): # TODO what if user sends in both type and actionid vs just actionid """Retrieve a list of Result objects. @@ -623,22 +749,25 @@ def getResults(self, ids=None, type=None, uuids=None, actionid=None, simulationi Args: ids (list, optional): List of ResultIDs. - type (str, optional): Type of Result from + restype (str, optional): Type of Result from `controlled vocabulary name `_. uuids (list, optional): List of UUIDs string. actionid (int, optional): ActionID. simulationid (int, optional): SimulationID. - sfid (int, optional): SamplingFeatureID. variableid (int, optional): VariableID. - siteid (int, optional): SiteID. + siteid (int, optional): SiteID. - goes through related features table and finds all of results + recorded at the given site + sfids(list, optional): List of Sampling Feature IDs integer. + sfuuids(list, optional): List of Sampling Feature UUIDs string. + sfcodes=(list, optional): List of Sampling Feature codes string. Returns: list: List of Result objects Examples: >>> ReadODM2.getResults(ids=[39,40]) - >>> ReadODM2.getResults(type='Time series coverage') - >>> ReadODM2.getResults(sfid=65) + >>> ReadODM2.getResults(restype='Time series coverage') + >>> ReadODM2.getResults(sfids=[65]) >>> ReadODM2.getResults(uuids=['a6f114f1-5416-4606-ae10-23be32dbc202', ... '5396fdf3-ceb3-46b6-aaf9-454a37278bb4']) >>> ReadODM2.getResults(simulationid=50) @@ -648,9 +777,13 @@ def getResults(self, ids=None, type=None, uuids=None, actionid=None, simulationi """ query = self._session.query(Results) - - if type: - query = query.filter_by(ResultTypeCV=type) + self._check_kwargs(['type', 'sfid'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the restype parameter instead.', + DeprecationWarning, stacklevel=2) + restype = kwargs['type'] + if restype: + query = query.filter_by(ResultTypeCV=restype) if variableid: query = query.filter_by(VariableID=variableid) if ids: @@ -658,14 +791,24 @@ def getResults(self, ids=None, type=None, uuids=None, actionid=None, simulationi if uuids: query = query.filter(Results.ResultUUID.in_(uuids)) if simulationid: - query = query.join(FeatureActions)\ - .join(Actions)\ - .join(Simulations)\ - .filter_by(SimulationID=simulationid) + query = query.join(FeatureActions) \ + .join(Actions) \ + .join(Simulations) \ + .filter_by(SimulationID=simulationid) if actionid: query = query.join(FeatureActions).filter_by(ActionID=actionid) - if sfid: - query = query.join(FeatureActions).filter_by(SamplingFeatureID=sfid) + if 'sfid' in kwargs: + warnings.warn('The parameter \'sfid\' is deprecated. ' + 'Please use the sfids parameter instead and send in a list.', + DeprecationWarning, stacklevel=2) + if kwargs['sfid']: + query = query.join(FeatureActions).filter_by(SamplingFeatureID=kwargs['sfid']) + if sfids or sfcodes or sfuuids: + sf_list = self.getSamplingFeatures(ids=sfids, codes=sfcodes, uuids=sfuuids) + sfids = [] + for sf in sf_list: + sfids.append(sf.SamplingFeatureID) + query = query.join(FeatureActions).filter(FeatureActions.SamplingFeatureID.in_(sfids)) if siteid: sfids = [x[0] for x in self._session.query( @@ -674,7 +817,14 @@ def getResults(self, ids=None, type=None, uuids=None, actionid=None, simulationi .join(RelatedFeatures.SamplingFeatureObj) .filter(RelatedFeatures.RelatedFeatureID == siteid) .all() - ] + ] + + # TODO does this code do the same thing as the code above? + # sf_list = self.getRelatedSamplingFeatures(rfid=siteid) + # sfids = [] + # for sf in sf_list: + # sfids.append(sf.SamplingFeatureID) + query = query.join(FeatureActions).filter(FeatureActions.SamplingFeatureID.in_(sfids)) try: @@ -684,22 +834,209 @@ def getResults(self, ids=None, type=None, uuids=None, actionid=None, simulationi return None # Datasets - def getDataSets(self, codes=None, uuids=None): + def getDataSets(self, ids=None, codes=None, uuids=None, dstype=None): """ - * Pass nothing - returns a list of all DataSet objects - * Pass a list of DataSetCode - returns a single DataSet object for each code - * Pass a list of UUIDS - returns a single DataSet object for each UUID + Retrieve a list of Datasets + + Args: + ids (list, optional): List of DataSetsIDs. + codes (list, optional): List of DataSet Codes. + uuids (list, optional): List of Dataset UUIDs string. + dstype (str, optional): Type of Dataset from + `controlled vocabulary name `_. + + + Returns: + list: List of DataSets Objects + + Examples: + >>> READ = ReadODM2(SESSION_FACTORY) + >>> READ.getDataSets(ids=[39, 40]) + >>> READ.getDataSets(codes=['HOME', 'FIELD']) + >>> READ.getDataSets(uuids=['a6f114f1-5416-4606-ae10-23be32dbc202', + ... '5396fdf3-ceb3-46b6-aaf9-454a37278bb4']) + >>> READ.getDataSets(dstype='singleTimeSeries') + """ q = self._session.query(DataSets) + if ids: + q = q.filter(DataSets.DataSetID.in_(ids)) + if codes: + q = q.filter(DataSets.DataSetCode.in_(codes)) + if uuids: + q.filter(DataSets.DataSetUUID.in_(uuids)) + if dstype: + q = q.filter(DataSets.DataSetTypeCV == dstype) + try: + return q.all() + except Exception as e: + print('Error running Query {}'.format(e)) + return None + + # Datasets + + def getDataSetsResults(self, ids=None, codes=None, uuids=None, dstype=None): + """ + Retrieve a detailed list of Datasets along with detailed metadata about the datasets + and the results contained within them + + **Must specify either DataSetID OR DataSetUUID OR DataSetCode)** + Args: + ids (list, optional): List of DataSetsIDs. + codes (list, optional): List of DataSet Codes. + uuids (list, optional): List of Dataset UUIDs string. + dstype (str, optional): Type of Dataset from + `controlled vocabulary name `_. + + + Returns: + list: List of DataSetsResults Objects + + Examples: + >>> READ = ReadODM2(SESSION_FACTORY) + >>> READ.getDataSetsResults(ids=[39, 40]) + >>> READ.getDataSetsResults(codes=['HOME', 'FIELD']) + >>> READ.getDataSetsResults(uuids=['a6f114f1-5416-4606-ae10-23be32dbc202', + ... '5396fdf3-ceb3-46b6-aaf9-454a37278bb4']) + >>> READ.getDataSetsResults(dstype='singleTimeSeries') + + """ + + # make sure one of the three arguments has been sent in + if all(v is None for v in [ids, codes, uuids]): + raise ValueError('Expected DataSetID OR DataSetUUID OR DataSetCode argument') + + q = self._session.query(DataSetsResults) \ + .join(DataSets) + if ids: + q = q.filter(DataSets.DataSetID.in_(ids)) if codes: q = q.filter(DataSets.DataSetCode.in_(codes)) if uuids: - q.q.filter(DataSets.DataSetUUID.in_(uuids)) + q.filter(DataSets.DataSetUUID.in_(uuids)) + if dstype: + q = q.filter(DataSets.DataSetTypeCV == dstype) try: return q.all() except Exception as e: print('Error running Query {}'.format(e)) + return None + + def getDataSetsValues(self, ids=None, codes=None, uuids=None, dstype=None, lowercols=True): + """ + Retrieve a list of datavalues associated with the given dataset info + + **Must specify either DataSetID OR DataSetUUID OR DataSetCode)** + Args: + ids (list, optional): List of DataSetsIDs. + codes (list, optional): List of DataSet Codes. + uuids (list, optional): List of Dataset UUIDs string. + dstype (str, optional): Type of Dataset from + `controlled vocabulary name `_. + lowercols (bool, optional): Make column names to be lowercase. + Default to True. + **Please start upgrading your code to rely on CamelCase column names, + In a near-future release, + the default will be changed to False, + and later the parameter may be removed**. + + + Returns: + list: List of Result Values Objects + + Examples: + >>> READ = ReadODM2(SESSION_FACTORY) + >>> READ.getDataSetsValues(ids=[39, 40]) + >>> READ.getDataSetsValues(codes=['HOME', 'FIELD']) + >>> READ.getDataSetsValues(uuids=['a6f114f1-5416-4606-ae10-23be32dbc202', + ... '5396fdf3-ceb3-46b6-aaf9-454a37278bb4']) + >>> READ.getDataSetsValues(dstype='singleTimeSeries', lowercols=False) + + """ + + dsr = self.getDataSetsResults(ids, codes, uuids, dstype) + + resids = [] + for ds in dsr: + resids.append(ds.ResultID) + + try: + return self.getResultValues(resultids=resids, lowercols=lowercols) + except Exception as e: + print('Error running Query {}'.format(e)) + return None + + def getSamplingFeatureDatasets(self, ids=None, codes=None, uuids=None, dstype=None, sftype=None): + """ + Retrieve a list of Datasets associated with the given sampling feature data. + + **Must specify either samplingFeatureID OR samplingFeatureUUID OR samplingFeatureCode)** + + Args: + ids (list, optional): List of SamplingFeatureIDs. + codes (list, optional): List of SamplingFeature Codes. + uuids (list, optional): List of UUIDs string. + dstype (str, optional): Type of Dataset from + `controlled vocabulary name `_. + sftype (str, optional): Type of SamplingFeature from + `controlled vocabulary name `_. + + Returns: + list: List of DataSetsResults Objects associated with the given sampling feature + + Examples: + >>> READ = ReadODM2(SESSION_FACTORY) + >>> READ.getSamplingFeatureDatasets(ids=[39, 40]) + >>> READ.getSamplingFeatureDatasets(codes=['HOME', 'FIELD']) + >>> READ.getSamplingFeatureDatasets(uuids=['a6f114f1-5416-4606-ae10-23be32dbc202', + ... '5396fdf3-ceb3-46b6-aaf9-454a37278bb4']) + >>> READ.getSamplingFeatureDatasets(dstype='singleTimeSeries') + >>> READ.getSamplingFeatureDatasets(sftype='Specimen') + + """ + + # make sure one of the three arguments has been sent in + if all(v is None for v in [ids, codes, uuids, sftype]): + raise ValueError( + 'Expected samplingFeatureID OR samplingFeatureUUID ' + 'OR samplingFeatureCode OR samplingFeatureType ' + 'argument') + + sf_query = self._session.query(SamplingFeatures) + if sftype: + sf_query = sf_query.filter(SamplingFeatures.SamplingFeatureTypeCV == sftype) + if ids: + sf_query = sf_query.filter(SamplingFeatures.SamplingFeatureID.in_(ids)) + if codes: + sf_query = sf_query.filter(SamplingFeatures.SamplingFeatureCode.in_(codes)) + if uuids: + sf_query = sf_query.filter(SamplingFeatures.SamplingFeatureUUID.in_(uuids)) + + sf_list = [] + for sf in sf_query.all(): + sf_list.append(sf) + + try: + sfds = [] + for sf in sf_list: + + q = self._session.query(DataSetsResults) \ + .join(Results) \ + .join(FeatureActions) \ + .filter(FeatureActions.SamplingFeatureID == sf.SamplingFeatureID) + + if dstype: + q = q.filter_by(DatasetTypeCV=dstype) + + vals = q.all() + + related = self.getRelatedSamplingFeatures(sf.SamplingFeatureID) + + sfds.append(SamplingFeatureDataSet(sf, vals, related)) + except Exception as e: + print('Error running Query: {}'.format(e)) return None + return sfds # Data Quality def getDataQuality(self): @@ -735,7 +1072,7 @@ def getResultsDataQuality(self): # TODO Equipment Schema Queries # Equipment - def getEquipment(self, codes=None, type=None, sfid=None, actionid=None): + def getEquipment(self, codes=None, equiptype=None, sfid=None, actionid=None, **kwargs): """ * Pass nothing - returns a list of all Equipment objects * Pass a list of EquipmentCodes- return a list of all Equipment objects that match each of the codes @@ -744,17 +1081,26 @@ def getEquipment(self, codes=None, type=None, sfid=None, actionid=None): * Pass an ActionID - returns a single Equipment object """ + self._check_kwargs(['type'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the equiptype parameter instead.', + DeprecationWarning, stacklevel=2) + equiptype = kwargs['type'] + + # NOTE: Equiptype currently unused! + if equiptype: + pass e = self._session.query(Equipment) if sfid: e = e.join(EquipmentUsed) \ - .join(Actions) \ - .join(FeatureActions) \ - .filter(FeatureActions.SamplingFeatureID == sfid) + .join(Actions) \ + .join(FeatureActions) \ + .filter(FeatureActions.SamplingFeatureID == sfid) if codes: e = e.filter(Equipment.EquipmentCode.in_(codes)) if actionid: e = e.join(EquipmentUsed).join(Actions) \ - .filter(Actions.ActionID == actionid) + .filter(Actions.ActionID == actionid) return e.all() def CalibrationReferenceEquipment(self): @@ -832,25 +1178,30 @@ def RelatedEquipment(self, code=None): return r.all() # Extension Properties - def getExtensionProperties(self, type=None): + def getExtensionProperties(self, exptype=None, **kwargs): """ * Pass nothing - return a list of all objects * Pass type- return a list of all objects of the given type """ # Todo what values to use for extensionproperties type + self._check_kwargs(['type'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the exptype parameter instead.', + DeprecationWarning, stacklevel=2) + exptype = kwargs['type'] e = ExtensionProperties - if type == 'action': + if exptype == 'action': e = ActionExtensionPropertyValues - elif type == 'citation': + elif exptype == 'citation': e = CitationExtensionPropertyValues - elif type == 'method': + elif exptype == 'method': e = MethodExtensionPropertyValues - elif type == 'result': + elif exptype == 'result': e = ResultExtensionPropertyValues - elif type == 'samplingfeature': + elif exptype == 'samplingfeature': e = SamplingFeatureExtensionPropertyValues - elif type == 'variable': + elif exptype == 'variable': e = VariableExtensionPropertyValues try: return self._session.query(e).all() @@ -859,28 +1210,33 @@ def getExtensionProperties(self, type=None): return None # External Identifiers - def getExternalIdentifiers(self, type=None): + def getExternalIdentifiers(self, eitype=None, **kwargs): """ * Pass nothing - return a list of all objects * Pass type- return a list of all objects of the given type """ + self._check_kwargs(['type'], kwargs) + if 'type' in kwargs: + warnings.warn('The parameter \'type\' is deprecated. Please use the eitype parameter instead.', + DeprecationWarning, stacklevel=2) + eitype = kwargs['type'] e = ExternalIdentifierSystems - if type.lowercase == 'citation': + if eitype.lowercase == 'citation': e = CitationExternalIdentifiers - elif type == 'method': + elif eitype == 'method': e = MethodExternalIdentifiers - elif type == 'person': + elif eitype == 'person': e = PersonExternalIdentifiers - elif type == 'referencematerial': + elif eitype == 'referencematerial': e = ReferenceMaterialExternalIdentifiers - elif type == 'samplingfeature': + elif eitype == 'samplingfeature': e = SamplingFeatureExternalIdentifiers - elif type == 'spatialreference': + elif eitype == 'spatialreference': e = SpatialReferenceExternalIdentifiers - elif type == 'taxonomicclassifier': + elif eitype == 'taxonomicclassifier': e = TaxonomicClassifierExternalIdentifiers - elif type == 'variable': + elif eitype == 'variable': e = VariableExternalIdentifiers try: return self._session.query(e).all() @@ -988,46 +1344,60 @@ def getResultDerivationEquations(self): """ return self._session.query(ResultDerivationEquations).all() - # Results - # ResultValues - def getResultValues(self, resultids, starttime=None, endtime=None): - """ - getResultValues(self, resultids, starttime=None, endtime=None) - * Pass in a list of ResultID - Returns a pandas dataframe object of type - that is specific to the result type - The resultids must be associated - with the same value type - * Pass a ResultID and a date range - returns a pandas dataframe object - of type that is specific to the result type with values between the input date range - * Pass a starttime - Returns a dataframe with the values after the given start time - * Pass an endtime - Returns a dataframe with the values before the given end time - - """ - type = self._session.query(Results).filter_by(ResultID=resultids[0]).first().ResultTypeCV - ResultType = TimeSeriesResults - if 'categorical' in type.lower(): - ResultType = CategoricalResultValues - elif 'measurement' in type.lower(): - ResultType = MeasurementResultValues - elif 'point' in type.lower(): - ResultType = PointCoverageResultValues - elif 'profile' in type.lower(): - ResultType = ProfileResultValues - elif 'section' in type.lower(): - ResultType = SectionResults - elif 'spectra' in type.lower(): - ResultType = SpectraResultValues - elif 'time' in type.lower(): - ResultType = TimeSeriesResultValues - elif 'trajectory' in type.lower(): - ResultType = TrajectoryResultValues - elif 'transect' in type.lower(): - ResultType = TransectResultValues - - q = self._session.query(ResultType).filter(ResultType.ResultID.in_(resultids)) + def getResultValues(self, resultids, starttime=None, endtime=None, lowercols=True): + """ + Retrieve result values associated with the given result. + + **The resultids must be associated with the same result type** + Args: + resultids (list): List of SamplingFeatureIDs. + starttime (object, optional): Start time to filter by as datetime object. + endtime (object, optional): End time to filter by as datetime object. + lowercols (bool, optional): Make column names to be lowercase. + Default to True. + **Please start upgrading your code to rely on CamelCase column names, + In a near-future release, + the default will be changed to False, + and later the parameter may be removed**. + + Returns: + DataFrame: Pandas dataframe of result values. + + Examples: + >>> READ = ReadODM2(SESSION_FACTORY) + >>> READ.getResultValues(resultids=[10, 11]) + >>> READ.getResultValues(resultids=[100, 20, 34], starttime=datetime.today()) + >>> READ.getResultValues(resultids=[1, 2, 3, 4], + >>> starttime=datetime(2000, 01, 01), + >>> endtime=datetime(2003, 02, 01), lowercols=False) + + """ + restype = self._session.query(Results).filter_by(ResultID=resultids[0]).first().ResultTypeCV + ResultValues = TimeSeriesResultValues + if 'categorical' in restype.lower(): + ResultValues = CategoricalResultValues + elif 'measurement' in restype.lower(): + ResultValues = MeasurementResultValues + elif 'point' in restype.lower(): + ResultValues = PointCoverageResultValues + elif 'profile' in restype.lower(): + ResultValues = ProfileResultValues + elif 'section' in restype.lower(): + ResultValues = SectionResults + elif 'spectra' in restype.lower(): + ResultValues = SpectraResultValues + elif 'time' in restype.lower(): + ResultValues = TimeSeriesResultValues + elif 'trajectory' in restype.lower(): + ResultValues = TrajectoryResultValues + elif 'transect' in restype.lower(): + ResultValues = TransectResultValues + + q = self._session.query(ResultValues).filter(ResultValues.ResultID.in_(resultids)) if starttime: - q = q.filter(ResultType.ValueDateTime >= starttime) + q = q.filter(ResultValues.ValueDateTime >= starttime) if endtime: - q = q.filter(ResultType.ValueDateTime <= endtime) + q = q.filter(ResultValues.ValueDateTime <= endtime) try: # F841 local variable 'vals' is assigned to but never used # vals = q.order_by(ResultType.ValueDateTime) @@ -1037,6 +1407,14 @@ def getResultValues(self, resultids, starttime=None, endtime=None): con=self._session_factory.engine, params=query.params ) + if not lowercols: + df.columns = [self._get_columns(ResultValues)[c] for c in df.columns] + else: + warnings.warn( + 'In a near-future release, ' + 'the parameter \'lowercols\' default will be changed to False, ' + 'and later the parameter may be removed.', + DeprecationWarning, stacklevel=2) return df except Exception as e: print('Error running Query: {}'.format(e)) @@ -1096,16 +1474,21 @@ def getModels(self, codes=None): print('Error running Query: {}'.format(e)) return None - def getRelatedModels(self, id=None, code=None): + def getRelatedModels(self, modid=None, code=None, **kwargs): """ getRelatedModels(self, id=None, code=None) * Pass a ModelID - get a list of converter objects related to the converter having ModelID * Pass a ModelCode - get a list of converter objects related to the converter having ModeCode """ + self._check_kwargs(['id'], kwargs) + if 'id' in kwargs: + warnings.warn('The parameter \'id\' is deprecated. Please use the modid parameter instead.', + DeprecationWarning, stacklevel=2) + modid = kwargs['id'] m = self._session.query(Models).select_from(RelatedModels).join(RelatedModels.ModelObj) - if id: - m = m.filter(RelatedModels.ModelID == id) + if modid: + m = m.filter(RelatedModels.ModelID == modid) if code: m = m.filter(Models.ModelCode == code) diff --git a/odm2api/ODM2/services/updateService.py b/odm2api/ODM2/services/updateService.py index 1bec3ce..87e77b8 100644 --- a/odm2api/ODM2/services/updateService.py +++ b/odm2api/ODM2/services/updateService.py @@ -5,7 +5,7 @@ from datetime import datetime from odm2api.ODM2 import serviceBase -from odm2api.ODM2.models import * +from odm2api.ODM2.models import (Actions, Results) # ################################################################################ @@ -13,39 +13,39 @@ # ################################################################################ class UpdateODM2(serviceBase): - def update(self, value): self._session.add(value) self._session.commit() return value -# ################################################################################ -# Core -# ################################################################################ + # ################################################################################ + # Core + # ################################################################################ def updateResultValidDateTime(self, resultId, dateTime): - #check type of "validdatetime' - #if not datetime do this: + # check type of "validdatetime' + # if not datetime do this: # dt = dateTime.to_datetime() - #else dt = dateTime + # else dt = dateTime if (type(dateTime) != datetime): dt = dateTime.to_datetime() else: dt = dateTime - q = self._session.query(Results).filter(Results.ResultID == int(resultId)).update({'ValidDateTime': dt}) + q = self._session.query(Results).filter(Results.ResultID == int(resultId)) + q.update({'ValidDateTime': dt}) + self._session.commit() def updateResult(self, resultID=None, valuecount=None, result=None): if resultID: q = self._session.query(Results).filter(Results.ResultID == int(resultID)) if valuecount: - q.update({"ValueCount": valuecount}) + q.update({'ValueCount': valuecount}) if result: self._session.add(result) self._session.commit() - - def updateAction(self, actionID=None, begin=None, end=None, action = None): + def updateAction(self, actionID=None, begin=None, end=None, action=None): if actionID: q = self._session.query(Actions).filter(Actions.ActionID == int(actionID)) # if (type(begin) != datetime): @@ -54,77 +54,40 @@ def updateAction(self, actionID=None, begin=None, end=None, action = None): # end = end.to_datetime() if begin: - q.update({"BeginDateTime": begin}) + q.update({'BeginDateTime': begin}) if end: - q.update({"EndDateTime": end}) + q.update({'EndDateTime': end}) elif action: self._session.add(action) self._session.commit() - # ################################################################################ # Data Quality # ################################################################################ - - - - # ################################################################################ # Equipment # ################################################################################ - - - - - # ################################################################################ # Extension Properties # ################################################################################ - - - - # ################################################################################ # External Identifiers # ################################################################################ - - - - # ################################################################################ # Lab Analyses # ################################################################################ - - - - # ################################################################################ # Provenance # ################################################################################ - - - - # ################################################################################ # Results # ################################################################################ - - - - # ################################################################################ # Sampling Features # ################################################################################ - - - - # ################################################################################ # Sensors # ################################################################################ - - - -# ################################################################################ +################################################################################ # ODM2 # ################################################################################ diff --git a/odm2api/base.py b/odm2api/base.py index a164a1d..6887017 100644 --- a/odm2api/base.py +++ b/odm2api/base.py @@ -1,5 +1,6 @@ from __future__ import (absolute_import, division, print_function) + class serviceBase(object): def __init__(self, session_factory, debug=False): """Must send in either a session_factory.""" @@ -42,10 +43,9 @@ def __repr__(self): if 'obj' in v.lower(): del valuedict[v] - if v == "_sa_instance_state": - del valuedict["_sa_instance_state"] - return "<%s(%s)>" % (self.__class__.__name__, str(valuedict)) - + if v == '_sa_instance_state': + del valuedict['_sa_instance_state'] + return '<%s(%s)>' % (self.__class__.__name__, str(valuedict)) class modelBase(): diff --git a/tests/test_connection.py b/tests/test_connection.py index e233eff..a8bfacf 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -2,7 +2,9 @@ from odm2api.ODM2.models import CVElevationDatum from odm2api.ODMconnection import dbconnection + import pytest + from sqlalchemy.engine import reflection diff --git a/tests/test_odm2/test_readservice.py b/tests/test_odm2/test_readservice.py index 6ea9154..f1189aa 100644 --- a/tests/test_odm2/test_readservice.py +++ b/tests/test_odm2/test_readservice.py @@ -11,7 +11,6 @@ import sqlalchemy from sqlalchemy.orm import class_mapper - globals_vars = {} @@ -75,8 +74,7 @@ def setup(self): self.engine = globals_vars['engine'] self.db = globals_vars['db'] - -# Sampling Features + # Sampling Features def test_getAllSamplingFeatures(self): # get all models from the database res = self.engine.execute('SELECT * FROM SamplingFeatures').fetchall() @@ -92,17 +90,120 @@ def test_getSamplingFeatureByID(self): resapi = self.reader.getSamplingFeatures(ids=[sfid]) assert resapi is not None -# Models - """ - TABLE Models - ModelID INTEGER NOT NULL PRIMARY KEY, - ModelCode VARCHAR (50) NOT NULL, - ModelName VARCHAR (255) NOT NULL, - ModelDescription VARCHAR (500) NULL, - Version VARCHAR (255) NULL, - ModelLink VARCHAR (255) NULL - """ + def test_getSamplingFeatureByCode(self): + # get all models from the database + res = self.engine.execute('SELECT * FROM SamplingFeatures').fetchone() + code = res[2] + # get all simulations using the api + resapi = self.reader.getSamplingFeatures(codes=[code]) + assert resapi is not None + + # DataSets + def test_getDataSets(self): + # get all datasets from the database + ds = self.engine.execute('SELECT * FROM DataSets').fetchone() + dsid = ds[0] + + dsapi = self.reader.getDataSets(ids=[dsid]) + assert dsapi is not None + assert True + + def test_getDataSetsResults(self): + # get all datasetresults from the database + dsr = self.engine.execute('SELECT * FROM DataSetsResults').fetchone() + dsid = dsr[2] + + dsrapi = self.reader.getDataSetsResults(ids=[dsid]) + assert dsrapi is not None + assert True + + def test_getDataSetsValues(self): + + dsr = self.engine.execute('SELECT * FROM DataSetsResults').fetchone() + dsid = dsr[2] + + values = self.reader.getDataSetsValues(ids=[dsid]) + assert values is not None + assert len(values) > 0 + + def test_getSamplingFeatureDataSets(self): + try: + # find a sampling feature that is associated with a dataset + sf = self.engine.execute( + 'SELECT * from SamplingFeatures as sf ' + 'inner join FeatureActions as fa on fa.SamplingFeatureID == sf.SamplingFeatureID ' + 'inner join Results as r on fa.FeatureActionID == r.FeatureActionID ' + 'inner join DataSetsResults as ds on r.ResultID == ds.ResultID ' + ).fetchone() + assert len(sf) > 0 + + # get the dataset associated with the sampling feature + ds = self.engine.execute( + 'SELECT * from DataSetsResults as ds ' + 'inner join Results as r on r.ResultID == ds.ResultID ' + 'inner join FeatureActions as fa on fa.FeatureActionID == r.FeatureActionID ' + 'where fa.SamplingFeatureID = ' + str(sf[0]) + ).fetchone() + assert len(ds) > 0 + + print(sf[0]) + # get the dataset associated with the sampling feature using hte api + dsapi = self.reader.getSamplingFeatureDatasets(ids=[sf[0]]) + + assert dsapi is not None + assert len(dsapi) > 0 + assert dsapi[0].datasets is not None + assert dsapi[0].SamplingFeatureID == sf[0] + # assert ds[0] == dsapi[0] + except Exception as ex: + print(ex) + assert False + finally: + self.reader._session.rollback() + + # Results + def test_getAllResults(self): + + # get all results from the database + res = self.engine.execute('SELECT * FROM Results').fetchall() + print(res) + # get all results using the api + resapi = self.reader.getResults() + assert len(res) == len(resapi) + + def test_getResultsByID(self): + # get a result from the database + res = self.engine.execute('SELECT * FROM Results').fetchone() + resultid = res[1] + + # get the result using the api + resapi = self.reader.getResults(ids=[resultid]) + assert resapi is not None + + def test_getResultsBySFID(self): + sf = self.engine.execute( + 'SELECT * from SamplingFeatures as sf ' + 'inner join FeatureActions as fa on fa.SamplingFeatureID == sf.SamplingFeatureID ' + 'inner join Results as r on fa.FeatureActionID == r.FeatureActionID ' + ).fetchone() + assert len(sf) > 0 + sfid = sf[0] + + res = self.engine.execute( + 'SELECT * from Results as r ' + 'inner join FeatureActions as fa on fa.FeatureActionID == r.FeatureActionID ' + 'where fa.SamplingFeatureID = ' + str(sfid) + ).fetchone() + + assert len(res) > 0 + # get the result using the api + resapi = self.reader.getResults(sfids=[sfid]) + assert resapi is not None + assert len(resapi) > 0 + assert resapi[0].ResultID == res[0] + + # Models def test_getAllModels(self): # get all models from the database res = self.engine.execute('SELECT * FROM Models').fetchall() @@ -118,20 +219,7 @@ def test_getModelByCode(self): resapi = self.reader.getModels(codes=[modelCode]) assert resapi is not None - -# RelatedModels - """ - TABLE RelatedModels ( - RelatedID INTEGER NOT NULL PRIMARY KEY, - ModelID INTEGER NOT NULL, - RelationshipTypeCV VARCHAR (255) NOT NULL, - RelatedModelID INTEGER NOT NULL, - FOREIGN KEY (RelationshipTypeCV) REFERENCES CV_RelationshipType (Name) - ON UPDATE NO ACTION ON DELETE NO ACTION, - FOREIGN KEY (ModelID) REFERENCES Models (ModelID) - ON UPDATE NO ACTION ON DELETE NO ACTION - """ - + # RelatedModels def test_getRelatedModelsByID(self): # get related models by id using the api resapi = self.reader.getRelatedModels(id=1) @@ -143,7 +231,7 @@ def test_getRelatedModelsByCode(self): resapi = self.reader.getRelatedModels(code='swat') assert resapi is not None assert len(resapi) > 0 - print(resapi[0].ModelCode) + # print(resapi[0].ModelCode) assert resapi[0].ModelCode == 'swat' # test converter code that doesn't exist resapi = self.reader.getRelatedModels(code='None') @@ -155,60 +243,7 @@ def test_getRelatedModelsByCode(self): resapi = self.reader.getRelatedModels(code=234123) assert not resapi - -# Results - """ - TABLE Results ( - ResultID INTEGER NOT NULL PRIMARY KEY, - ResultUUID VARCHAR(36) NOT NULL, - FeatureActionID INTEGER NOT NULL, - ResultTypeCV VARCHAR (255) NOT NULL, - VariableID INTEGER NOT NULL, - UnitsID INTEGER NOT NULL, - TaxonomicClassifierID INTEGER NULL, - ProcessingLevelID INTEGER NOT NULL, - ResultDateTime DATETIME NULL, - ResultDateTimeUTCOffset INTEGER NULL, - ValidDateTime DATETIME NULL, - ValidDateTimeUTCOffset INTEGER NULL, - StatusCV VARCHAR (255) NULL, - SampledMediumCV VARCHAR (255) NOT NULL, - ValueCount INTEGER NOT NULL - """ - def test_getAllResults(self): - # get all results from the database - res = self.engine.execute('SELECT * FROM Results').fetchall() - print(res) - # get all results using the api - resapi = self.reader.getResults() - assert len(res) == len(resapi) - - def test_getResultsByID(self): - # get a result from the database - res = self.engine.execute('SELECT * FROM Results').fetchone() - resultid = res[1] - - # get the result using the api - resapi = self.reader.getResults(ids=[resultid]) - assert resapi is not None - -# Simulations - """ - TABLE Simulations ( - SimulationID INTEGER NOT NULL PRIMARY KEY, - ActionID INTEGER NOT NULL, - SimulationName VARCHAR (255) NOT NULL, - SimulationDescription VARCHAR (500) NULL, - SimulationStartDateTime DATETIME NOT NULL, - SimulationStartDateTimeUTCOffset INTEGER NOT NULL, - SimulationEndDateTime DATETIME NOT NULL, - SimulationEndDateTimeUTCOffset INTEGER NOT NULL, - TimeStepValue FLOAT NOT NULL, - TimeStepUnitsID INTEGER NOT NULL, - InputDataSetID INTEGER NULL, - ModelID INTEGER NOT NULL, - """ - + # Simulations def test_getAllSimulations(self): # get all simulation from the database res = self.engine.execute('SELECT * FROM Simulations').fetchall() @@ -246,7 +281,7 @@ def test_getResultsBySimulationID(self): ).first() assert len(res) > 0 res = rawSql2Alchemy(res, models.Results) - print(res) + # print(res) # get simulation by id using the api # resapi = self.reader.getResultsBySimulationID(simulation.SimulationID)