From fc7a105b341807c639090fab38e9e93f53600617 Mon Sep 17 00:00:00 2001 From: iamcourtney Date: May 09 2016 12:33:43 +0000 Subject: [PATCH 1/11] updating files --- diff --git a/databaseUtilities.py b/databaseUtilities.py index efda0ba..e5291aa 100644 --- a/databaseUtilities.py +++ b/databaseUtilities.py @@ -1,6 +1,9 @@ -import sqlite3 as lite -import sys +import pymongo +from pymongo import MongoClient +from bs4 import BeautifulSoup +from urllib import urlopen import json +import lxml class ModularityDb: @@ -8,210 +11,60 @@ class ModularityDb: ''' def Initialize - @purpose: [Private] Initialize the database connection + @purpose: [Private] Initialize the database @input: db_name: name of the database (string) @variables: - connection: sqlite connection - cursor: sqlite curser object - table_exists: does the modularity table exist? true/false + client: mongoDb client + db: database + collection: mongo collection + posts: mongoDb posts ''' - def __init__(self, db_name): - self.connection = lite.connect(db_name) - self.cursor = self.connection.cursor() - self.table_exists = False - - #Initialize modularity table - self.__initializeModularityTable() - - ''' - def createTable - - @purpose: [Private] Creates a table in the database - - @input: - tbl_name: name of the table (string) - columns: column titles (list of strings) - ''' - def __createTable(self, columns, column_types): - - if self.table_exists == True: - return; - - n_columns = len(columns) - n_column_types = len(column_types) - - if (n_columns == n_column_types): - - #Build command - cmd = "CREATE TABLE modularity(" - for xx in range(0,n_columns): - - cmd = cmd + columns[xx] + " " - if (xx < n_columns - 1): - cmd = cmd + column_types[xx] + ", " - else: - cmd = cmd + column_types[xx] + ")" - - #Create table - with self.connection: - try: - self.cursor.execute(cmd) - except: - pass - - #Mark table as created - self.table_exists = True - - else: - print "Could not create table. len(columns) != len(column_types)" - - ''' - def __initializeModularityTable - - @purpose: [Private] to initialize modularity table - ''' - def __initializeModularityTable(self): - columns = ["Id", "name","url","license","version","upstreamDocs","upstreamCommunity","depModules","componentRPMs"] - dtypes = ["INT", "TEXT", "TEXT", "TEXT", "TEXT", "TEXT", "TEXT", "TEXT", "TEXT"] - self.__createTable( columns , dtypes ) - + def __init__(self): + self.client = MongoClient('localhost', 27017) + self.db = self.client.ModularityDb + self.collection = self.db.ModularityDb + self.posts = self.db.posts + self.packages = {} + self.__pullData() ''' - def cleanup + def __pullData - @purpose: Remove the autogenerated 'modularity' table + @purpose: [Private] Pull module data from database ''' - def cleanup(self): - cmd = "DROP TABLE IF EXISTS modularity" - with self.connection: - self.cursor.execute(cmd) - self.table_exists = False + def __pullData(self): + r = urlopen('https://admin.stg.fedoraproject.org/pkgdb/api/packages?namespace=modules') + soup = BeautifulSoup(r,"lxml") + text = soup.getText() + json_data = json.loads(text) + self.packages = [json_data['packages']][0] + print self.packages + self.posts.insert(self.packages) ''' - def insertRow + def printDatabase - @purpose: Insert row of data to sqllite database + @purpose: print out entire database @inputs: tbl_name: name of the table (string) - values: values to insert (list) ''' - def insertRow(self, values): - #Build command - cmd = "INSERT INTO modularity VALUES(" + \ - ", ".join(["?" for i in range(0, len(values))]) + ")" - - #Insert - with self.connection: - self.cursor.execute(cmd, values) + def printDatabase(self): + result = self.getDatabase() + print result ''' - def insertJSON - - @purpose: Insert JSON data into database. JSON data must have the following - format: + def getDatabase - { : { - 'Upstream Community': , - 'Dependent Modules': , - 'License': , - 'URL': , - 'Version': , - 'Upstream Docs': , - 'Component RPMs': - } - } + @purpose: Retrieve entire database as list of dictionary objects - @inputs: - json_data: The data you wish to enter, in json format. - id: List of SQL ids for the data + @return: list of dictionary objects ''' - def insertJSON(self,json_data,id): - parsed_json = json.loads(json_data); - - data_list = [] - tmp_list = [] - xx = 0 - - for repo_name in parsed_json: - temp_dict = parsed_json[repo_name] - for metadata_type in temp_dict: - if (metadata_type == 'URL'): - url = temp_dict[metadata_type] - elif (metadata_type == 'License'): - license = temp_dict[metadata_type] - elif (metadata_type == 'Version'): - version = temp_dict[metadata_type] - elif (metadata_type == 'Upstream Docs'): - docs = temp_dict[metadata_type] - elif (metadata_type == 'Upstream Community'): - community = temp_dict[metadata_type] - elif (metadata_type == 'Dependent Modules'): - tmp = temp_dict[metadata_type] - dep_modules = ' '.join(tmp) - else: - tmp = temp_dict[metadata_type] - component_rpms = ' '.join(tmp) - - tmp_list = [id[xx],repo_name,url,license,version,docs,community,dep_modules,component_rpms] - data_list.append(tuple(tmp_list)) - - xx = xx + 1 - tmp_list = [] - - input_tuple = tuple(data_list) - for row in input_tuple: - self.insertRow(row) - - - ''' - def printTable - - @purpose: print out an entire table - - @inputs: - tbl_name: name of the table (string) - ''' - def printTable(self): - table = self.getTable() - print table - - ''' - def getTable - - @purpose: Retrieve entire table as dictionary object - - @inputs: - tbl_name: name of the table (string) - - @return: dictionary oject - ''' - def getTable(self): - - modularity_table = {} - - if (self.table_exists == False): - print "Modularity table does not exist." - - with self.connection: - self.cursor.execute("SELECT * FROM modularity") - rows = self.cursor.fetchall() - - for row in rows: - temp = {} - temp['URL'] = row[2] - temp['License'] = row[3] - temp['Version'] = row[4] - temp['Upstream Docs'] = row[5] - temp['Upstream Community'] = row[6] - temp['Dependent Modules'] = row[7].split(" ") - temp['Component RPMs'] = row[8].split(" ") - modularity_table[row[1]] = temp - - return modularity_table + def getDatabase(self): + return self.packages ''' def query @@ -219,128 +72,15 @@ class ModularityDb: @purpose: Query modularity table @inputs: - column: Data column to select + query_dict: dictionary of data to search data: data to search for - ''' - def query(self, column, data): - - result = {} - mapping = { - "id" : "Id", - "name" : "name", - "url" : "URL", - "license" : "license", - "version" : "version", - "upstreamdocs" : "upstreamDocs", - "upstreamcommunity" : "upstreamCommunity", - "depmodules" : "depModules", - "componentrpms" : "componentRPMs" - } - - if (self.table_exists == True): - cmd = "SELECT * FROM modularity WHERE " + mapping[column] - if column in ('depmodules', 'componentrpms'): - cmd = cmd + " LIKE ?" - data = "%" + data + "%" - else: - cmd = cmd + " = ?" - with self.connection: - self.cursor.execute(cmd, [ data ]) - rows = self.cursor.fetchall() + @return: + list of dictionary objects - for row in rows: - temp = {} - temp['URL'] = row[2] - temp['License'] = row[3] - temp['Version'] = row[4] - temp['Upstream Docs'] = row[5] - temp['Upstream Community'] = row[6] - temp['Dependent Modules'] = row[7].split(" ") - temp['Component RPMs'] = row[8].split(" ") - result[row[1]] = temp - - return result - - ''' - def queryOR - - @purpose: Query modularity table using OR - - @inputs: - column: Data column to select - data: data to search for - - EXAMPLE: queryOR('Id',1,'Id',2) -- Finds all data where Id=1 *or* Id=2 ''' - def queryOR(self,*args): - - result = {} - n_args = len(args) - - if (self.table_exists == False): - print "Modularity table does not exist" - - elif (n_args % 2 == 1): - print "Invalid arguments" - - else: - for xx in xrange(0,n_args,2): - query_res = self.query(args[xx],args[xx+1]) - result.update(query_res) - - return result - - ''' - def queryAND - - @purpose: Query modularity table using AND - - @inputs: - column: Data column to select - data: data to search for - - EXAMPLE: queryAND('version',1,'name','Langdon') -- Finds all data where - version=1 *and* name='Langdon' - ''' - def queryAND(self,*args): - - result = {} - n_args = len(args) - - if self.table_exists == False: - print "Modularity table does not exist" - - elif (n_args < 2) or (n_args % 2 == 1): - print "Invalid arguments" - - else: - # Get a pair of values from the list, no std. way? - def _get2(vals): - have_old = False - oval = None - for val in vals: - if have_old: - have_old = False - yield oval, val - else: - have_old = True - oval = val - - found = None - for key,val in _get2(args): - res = self.query(key, val) - - qfound = set(res.keys()) - if found is None: - found = qfound - else: - found.intersection_update(qfound) - if not found: # No commonality - break - result = {} - for x in found: - result[x] = res[x] - # result = {x:res[x] for x in found} - + def query(self, query_dict): + result = [] + for post in self.posts.find(query_dict): + result.append(post) return result diff --git a/demo.py b/demo.py index fe3d8ae..fdab030 100644 --- a/demo.py +++ b/demo.py @@ -1,56 +1,16 @@ from databaseUtilities import ModularityDb -#Initialize database (this creates a table called 'modularity') -myDb = ModularityDb("modularityTest.db") +#Initialize database +myDb = ModularityDb() -#Insert fake data into table, as tuple -data = ( - (1, "walrus", "www.walrus.com", "ABC_Lic", 1, "doc", "com", "kitty", "ABC.rpm DEF.rpm GHI.rpm"), - (2, "kitty", "www.penny.com", "ABC_Lic", 7, "doc", "com", "Langdon", "ABC.rpm"), - (3, "Courtney", "www.Courtney.com", "ABC_Lic", 13, "doc", "com", "kitty", "ABC.rpm"), - (4, "Langdon", "www.Langdon.com", "ABC_Lic", 1, "doc", "com", "walrus kitty", "ABC.rpm"), - (5, "James", "www.James.com", "ABC_Lic", 4, "doc", "com", "walrus", "ABC.rpm"), - (6, "Jan", "www.Jan.com", "ABC_Lic", 1, "doc", "com", "kitty", "ABC.rpm"), - (7, "Petr", "www.Petr.com", "ABC_Lic", 3, "doc", "com", "kitty", "ABC.rpm"), -) +#Print database +myDb.printDatabase() -for row in data: - myDb.insertRow( row ) +#Test Queries +q1 = {"name":"testmodule"} +q2 = {"name":"doesnotexist"} +q3 = {"koschei_monitor" : True} -#Insert fake data into table, as JSON -test_json = ''' -{"yakuake": {"Upstream Community": "http://yakuake.kde.org/", "Dependent Modules": ["core"], "License": "GPLv2 or GPLv3", "URL": "https://copr.fedorainfracloud.org/coprs/rdieter/yakuake/repo/fedora-23/rdieter-yakuake-fedora-23.repo", "Version": "3.0.2-1", "Upstream Docs": "https://extragear.kde.org/apps/yakuake/", "Component RPMs": ["yakuake-3.0.2-1.fc23", "yakuake-debuginfo-3.0.2-1.fc23"]}, "howdoi": {"Upstream Community": "https://pypi.python.org/pypi/howdoi", "Dependent Modules": ["core"], "License": "MIT", "URL": "https://copr.fedorainfracloud.org/coprs/psabata/howdoi/repo/fedora-23/psabata-howdoi-fedora-23.repo", "Version": "1.1.7-1", "Upstream Docs": "https://pypi.python.org/pypi/howdoi", "Component RPMs": ["python2-howdoi-1.1.7-2.fc23"]}, "httpd24": {"Upstream Community": "http://httpd.apache.org/", "Dependent Modules": ["core"], "License": "ASL 2.0", "URL": "https://copr.fedorainfracloud.org/coprs/psabata/httpd24/repo/fedora-23/psabata-httpd24-fedora-23.repo", "Version": "2.4.18-1", "Upstream Docs": "http://httpd.apache.org/docs/2.4/", "Component RPMs": ["httpd-2:2.4.18-3.fc23", "httpd-debuginfo-2:2.4.18-3.fc23", "httpd-devel-2:2.4.18-3.fc23", "httpd-filesystem-2:2.4.18-3.fc23", "httpd-manual-2:2.4.18-3.fc23", "httpd-tools-2:2.4.18-3.fc23", "mod_ldap-2:2.4.18-3.fc23", "mod_proxy_html-2:2.4.18-3.fc23", "mod_session-2:2.4.18-3.fc23", "mod_ssl-2:2.4.18-3.fc23"]}, "httpd22": {"Upstream Community": "http://httpd.apache.org/", "Dependent Modules": ["core"], "License": "ASL 2.0", "URL": "https://copr.fedorainfracloud.org/coprs/jkaluza/httpd22/repo/fedora-23/jkaluza-httpd22-fedora-23.repo", "Version": "2.2.31-1", "Upstream Docs": "http://httpd.apache.org/docs/2.2/", "Component RPMs": ["httpd-2:2.2.31-4.fc23", "httpd-debuginfo-2:2.2.31-4.fc23", "httpd-devel-2:2.2.31-4.fc23", "httpd-manual-2:2.2.31-4.fc23", "httpd-tools-2:2.2.31-4.fc23", "mod_ssl-2:2.2.31-4.fc23"]}, "rocketchat": {"Upstream Community": "https://rocket.chat/", "Dependent Modules": ["core"], "License": "MIT", "URL": "https://copr.fedorainfracloud.org/coprs/mosquito/rocketchat/repo/fedora-23/mosquito-rocketchat-fedora-23.repo", "Version": "1.2.0-1", "Upstream Docs": "https://rocket.chat/", "Component RPMs": ["rocketchat-1.2.0-1.gitabb7b81.fc23"]}, "n1": {"Upstream Community": "https://nylas.com/N1/", "Dependent Modules": ["core"], "License": "MIT", "URL": "https://copr.fedorainfracloud.org/coprs/mosquito/n1/repo/fedora-23/mosquito-n1-fedora-23.repo", "Version": "0.4.14-1", "Upstream Docs": "https://nylas.com/N1/docs/", "Component RPMs": ["n1-0.4.14-1.git53cd69b.fc23"]}, "nvi": {"Upstream Community": "http://www.bostic.com/vi/", "Dependent Modules": ["core"], "License": "BSD", "URL": "https://copr.fedorainfracloud.org/coprs/mcepl/nvi/repo/fedora-23/mcepl-nvi-fedora-23.repo", "Version": "1.81.6-1", "Upstream Docs": "https://sites.google.com/a/bostic.com/keithbostic/vi/nvi-faq", "Component RPMs": ["nvi-1.81.6-13.fc23", "nvi-debuginfo-1.81.6-13.fc23"]}}''' - -myDb.insertJSON(test_json,[7,8,9,10,11,12,13]) - -#Print table -myDb.printTable() -print "\n" - -#Get table as dictionary -modularity_table = myDb.getTable() -print modularity_table -print "\n" - -#Basic query -result = myDb.query('depModules','walrus') -print result -print "\n" - -#Query 'or' (find which modules have Id=1 *or* version=13) -result = myDb.queryOR('Id',1,'version',13) -print result -print "\n" - -#Query 'and' (find which modules have version=1 *and* name='Langdon') -result = myDb.queryAND('version',1,'name','Langdon') -print result -print "\n" - -#Query 'and' (find which modules have version=1 *and* name='Langdon' *and* url='abc') -result = myDb.queryAND('version',1,'name','Langdon','url','abc') -print result -print "\n" - -#Cleanup (remove autogenerated 'modularity' table) -myDb.cleanup() +print "q1:\n", myDb.query(q1) +print "q2:\n",myDb.query(q2) #empty result! +print "q3:\n", myDb.query(q3) From b107306653eaf7031653a48c3579f579df90b54a Mon Sep 17 00:00:00 2001 From: iamcourtney Date: May 09 2016 12:53:49 +0000 Subject: [PATCH 2/11] Modifying fm.py --- diff --git a/databaseUtilities.py b/databaseUtilities.py index e5291aa..3f68b76 100644 --- a/databaseUtilities.py +++ b/databaseUtilities.py @@ -41,7 +41,6 @@ class ModularityDb: text = soup.getText() json_data = json.loads(text) self.packages = [json_data['packages']][0] - print self.packages self.posts.insert(self.packages) ''' diff --git a/fm.py b/fm.py index fe3f711..d1157e5 100644 --- a/fm.py +++ b/fm.py @@ -24,17 +24,17 @@ if not args['path']: sys.exit(1) path = args['path'][0] -myDb = ModularityDb("modularityTest.db") +myDb = ModularityDb() import json if path == '/list': ret = [] - ret.append(myDb.getTable()) + ret.append(myDb.getDatabase()) print json.dumps(ret) elif path.startswith("/info/"): name = path[len("/info/"):] - ret = myDb.query('name',name) + ret = myDb.getDatabase() if bool(ret) == False: print json.dumps({'error' : 'not found'}) sys.exit(1) diff --git a/modularityTest.db b/modularityTest.db deleted file mode 100644 index 2b9fd19..0000000 Binary files a/modularityTest.db and /dev/null differ From f0bd2e0fa9fa15133d8f1109add03f81e740b9f9 Mon Sep 17 00:00:00 2001 From: iamcourtney Date: May 10 2016 12:49:40 +0000 Subject: [PATCH 3/11] Modifying scripts to use YAML files --- diff --git a/databaseUtilities.py b/databaseUtilities.py index 3f68b76..37d00db 100644 --- a/databaseUtilities.py +++ b/databaseUtilities.py @@ -1,9 +1,7 @@ import pymongo from pymongo import MongoClient -from bs4 import BeautifulSoup -from urllib import urlopen import json -import lxml +import yaml class ModularityDb: @@ -28,20 +26,19 @@ class ModularityDb: self.collection = self.db.ModularityDb self.posts = self.db.posts self.packages = {} - self.__pullData() + self.__getData() ''' - def __pullData + def __getData - @purpose: [Private] Pull module data from database + @purpose: [Private] get module data from yaml ''' - def __pullData(self): - r = urlopen('https://admin.stg.fedoraproject.org/pkgdb/api/packages?namespace=modules') - soup = BeautifulSoup(r,"lxml") - text = soup.getText() - json_data = json.loads(text) - self.packages = [json_data['packages']][0] - self.posts.insert(self.packages) + def __getData(self): + with open("metadata.yaml","r") as stream: + try: + self.packages = yaml.load(stream) + except yaml.YAMLError as exc: + print(exc) ''' def printDatabase @@ -83,3 +80,12 @@ class ModularityDb: for post in self.posts.find(query_dict): result.append(post) return result + + ''' + def cleanup + + @purpose: Clear the database + ''' + def cleanup(self): + C = Connection() + c.drop_database('ModularityDb') diff --git a/metadata.yaml b/metadata.yaml new file mode 100644 index 0000000..5e101f4 --- /dev/null +++ b/metadata.yaml @@ -0,0 +1,119 @@ +# Document type identifier +document: modulemd +# Module metadata format version +version: 0 +data: + # Module name, required + # NOTE: Module names might be structured differently later to include the + # module's vendor ecosystem, e.g. org.fedoraproject.foo in this case. + # This would also affect the dependency sections defined furhter below. + name: foo + # NOTE: Module versioning is currently being investigated + # http://taiga.fedorainfracloud.org/project/modularity/us/175 + # Module version, required + # Typically the same as the version of the main component, where applicable + version: 1.23 + # Module release, required + # Version of the module itself + release: 4 + # A short summary describing the module, required + summary: An example module + # A verbose description of the module, required + description: > + A module for the demonstration of the metadata format. Also, + the obligatory lorem ipsum dolor sit amet goes right here. + # Module and content licenses in the Fedora license identifier format, required + license: + # Module license, required + # This list covers licenses used for the module metadata, + # SPEC files or extra patches + module: + - MIT + # Content license, optional + # A list of licenses used by the packages in the module. + # This should be populated by build tools. + content: + - Beerware + - GPLv2+ + - zlib + # Extensible metadata block + # http://taiga.fedorainfracloud.org/project/modularity/us/135 + # A dictionary of user-defined keys and values. May be null. + # Optional. Defaults to null. + xmd: ~ + # Module dependencies, if any. Optional. + # NOTE: Module dependencies are currently being investigated + # http://taiga.fedorainfracloud.org/project/modularity/us/170 + # TODO: Provides, conflicts, obsoletes, recommends, etc. + # TODO: Support for comparison operators + dependencies: + # Build dependencies of this module, optional + # Keys are module names, values are the minimum required versions + # These modules define the buildroot for this module + buildrequires: + core: 23 + c-build: 6.0 + # Run-time dependencies of this module, optional + # Keys are module names, values are the minimum required versions + requires: + core: 23 + # References to external resources, typically upstream, optional + references: + # Upstream community website, if it exists, optional + community: http://www.example.com/ + # Upstream documentation, if it exists, optional + documentation: http://www.example.com/ + # Upstream bug tracker, if it exists, optional + tracker: http://www.example.com/ + # Functional components of the module, optional + components: + # RPM content of the module, optional + # NOTE: This is currently being investigated + # TODO: Package priority build order support (chainbuild) + # TODO: Binary package filtering (inclusive or exclusive) support + # TODO: Component tags: + # http://taiga.fedorainfracloud.org/project/modularity/us/186 + # http://taiga.fedorainfracloud.org/project/modularity/us/187 + # http://taiga.fedorainfracloud.org/project/modularity/us/188 + # TODO: Define architectures to build for + rpms: + # Should this module include dependencies of its RPM packages + # that are not provided by any of the modules it depends on? + # Optional, defaults to True + dependencies: True + # Should this module include all subpackages generated from the + # source packages? True if yes, False if only the main packages + # should be included. + # Optional, defaults to True + fulltree: True + # RPM-based packages of this module. + # Keys are the VCS/SRPM names, values dictionaries holding + # additional information where required, or null. + # Optional + packages: + bar: + # Use this repository if it's different from + # the build system configuration. + # Optional. + repository: https://pagure.io/bar.git + # Use this lookaside cache if it's different + # from the build system configuration. + # Optional. + cache: https://example.com/cache + # Use this specific commit ID for the build. + # If no commit ID is given, the latest master + # commit will be used. + # Optional. + commit: 26ca0c0 + # baz has no extra options + baz: ~ + xxx: + # xxx is only available on the listed architectures. + # Optional, defaults to all available arches. + arches: [ i686, x86_64 ] + # a list of architectures with multilib + # installs, i.e. both i686 and x86_64 + # versions will be installed on x86_64. + # Optional, defaults to no multilib. + multilib: [ x86_64 ] + xyz: ~ From 876cd68df182e552f19550c0d65532c4761dee6f Mon Sep 17 00:00:00 2001 From: iamcourtney Date: May 10 2016 13:17:59 +0000 Subject: [PATCH 4/11] Updating demo & databaseUtilities files --- diff --git a/databaseUtilities.py b/databaseUtilities.py index 37d00db..a9eb1d7 100644 --- a/databaseUtilities.py +++ b/databaseUtilities.py @@ -26,7 +26,16 @@ class ModularityDb: self.collection = self.db.ModularityDb self.posts = self.db.posts self.packages = {} + self.__insertData() + + ''' + def __insertData + + @purpose: [Private] insert data into the mongo database + ''' + def __insertData(self): self.__getData() + self.posts.insert(self.packages) ''' def __getData @@ -79,7 +88,11 @@ class ModularityDb: result = [] for post in self.posts.find(query_dict): result.append(post) - return result + if not result: + result.append(list(self.collection.find(query_dict))) + return result + else: + return result ''' def cleanup @@ -87,5 +100,5 @@ class ModularityDb: @purpose: Clear the database ''' def cleanup(self): - C = Connection() + c = pymongo.Connection() c.drop_database('ModularityDb') diff --git a/demo.py b/demo.py index fdab030..200dfa2 100644 --- a/demo.py +++ b/demo.py @@ -7,10 +7,15 @@ myDb = ModularityDb() myDb.printDatabase() #Test Queries -q1 = {"name":"testmodule"} -q2 = {"name":"doesnotexist"} -q3 = {"koschei_monitor" : True} +q1 = {'document': 'modulemd'} +q2 = {'version':2000} +q3 = {'version': 0} +q4 = {'data.components.rpms.dependencies': True} print "q1:\n", myDb.query(q1) print "q2:\n",myDb.query(q2) #empty result! print "q3:\n", myDb.query(q3) +print "q4:\n", myDb.query(q4) + +#Clear database +myDb.cleanup() From 25bba9baf39e3653920dc2163b65a846ad428eea Mon Sep 17 00:00:00 2001 From: iamcourtney Date: May 10 2016 16:37:59 +0000 Subject: [PATCH 5/11] Updating tests. [Note: tests are incomplete at this point.] --- diff --git a/tests/init_tests.py b/tests/init_tests.py new file mode 100644 index 0000000..997a4f3 --- /dev/null +++ b/tests/init_tests.py @@ -0,0 +1,12 @@ +from testbase import * +import json + +class InitTests(FMDBTests): + + def test_database_setup_properly(self): + MongoClient('localhost', 27017) + db = client.ModularityDb + self.assertEqual(type(self.fm.db),type(db)) + + def test_packages_not_empty(self): + self.assertEqual(bool(self.fm.packages),True) diff --git a/tests/query_tests.py b/tests/query_tests.py index 84eb8e3..f3ab2bf 100644 --- a/tests/query_tests.py +++ b/tests/query_tests.py @@ -5,195 +5,87 @@ class QueryTest(FMDBTests): #Query, by name, for a single module that we know exists (e.g., httpd22) def test_q_name(self): - result = self.fm.query('name','walrus') - self.assertIn('walrus', result) + result = self.fm.query({'data.name':'foo'}) + self.assertIn('name', result['data']) + self.assertEqual('foo', result['data']['name']) #Query, by name, for a single module that doesn't exist def test_q_name_fail(self): - result = self.fm.query('name','xwalrus') - self.assertNotIn('xwalrus', result) - - #Query, by URL, for a single module that we know exists - def test_q_url(self): - result = self.fm.query('url', 'www.walrus.com') - self.assertIn('walrus', result) - - #Query, by URL, for a single module that doesn't exist - def test_q_url_fail(self): - result = self.fm.query('url', 'xxx.walrus.com') - self.assertNotIn('walrus', result) + result = self.fm.query({'data.name':'xfoo'}) self.assertEqual(0, len(result)) - #Query, by License, for a single module that we know exists - def test_q_license_abc(self): - result = self.fm.query('license', 'ABC_Lic') - self.assertIn('walrus', result) - self.assertIn('kitty', result) - self.assertIn('Courtney', result) - self.assertIn('Jan', result) - self.assertIn('Petr', result) - self.assertEqual(len(result), 5) - def test_q_license_beer(self): - result = self.fm.query('license', 'Beerware') - self.assertIn('Langdon', result) - self.assertEqual(len(result), 1) - def test_q_license_gpl(self): - result = self.fm.query('license', 'GPL') - self.assertIn('James', result) - self.assertEqual(len(result), 1) + #Query, by document, for a single module that we know exists + def test_q_document(self): + result = self.fm.query({'document':'modulemd'}) + self.assertIn('document', result) + self.assertEqual('modulemd', result['document']) - #Query, by License, for a single module that doesn't exist - def test_q_license_xyz(self): - result = self.fm.query('license', 'xyz') - self.assertEqual(len(result), 0) + #Query, by document, for a single module that doesn't exist + def test_q_document_fail(self): + result = self.fm.query({'document':'xmodulemd'}) + self.assertEqual(0, len(result)) - #Query, by Version, for a single module that we know exists - def test_q_ver_1(self): - result = self.fm.query('version', '1') - self.assertIn('Langdon', result) - self.assertIn('Jan', result) - self.assertIn('walrus', result) - self.assertEqual(len(result), 3) - def test_q_ver_3(self): - result = self.fm.query('version', '3') - self.assertIn('Petr', result) - self.assertEqual(result['Petr']['Version'], '3') - self.assertEqual(len(result), 1) - def test_q_ver_4(self): - result = self.fm.query('version', '4') - self.assertIn('James', result) - self.assertEqual(result['James']['Version'], '4') - self.assertEqual(len(result), 1) - def test_q_ver_13(self): - result = self.fm.query('version', '13') - self.assertIn('Courtney', result) - self.assertEqual(len(result), 1) + #Query, by version, for a single module that we know exists + def test_q_version(self): + result = self.fm.query({'version':0}) + self.assertIn('version', result) + self.assertEqual(0, result['version']) - #Query, by Version, for a single module that doesn't exist - def test_q_ver_0(self): - result = self.fm.query('version', '0') - self.assertEqual(len(result), 0) - def test_q_ver_abc(self): - result = self.fm.query('version', 'ABC_lic') - self.assertEqual(len(result), 0) + #Query, by version, for a single module that doesn't exist + def test_q_version_fail(self): + result = self.fm.query({'document':'xmodulemd'}) + self.assertEqual(0, len(result)) - #Query, by Upstream Docs, for a single module that we know exists + #Query, by references, for a single module that we know exists def test_q_up_docs(self): - result = self.fm.query('upstreamdocs', 'doc') - self.assertIn('James', result) - self.assertEqual(result['James']['Upstream Docs'], 'doc') - - #Query, by Upstream Docs, for a single module that doesn't exist - def test_q_up_docs_nonexistent(self): - result = self.fm.query('upstreamdocs', 'non-existent') - self.assertEqual(len(result), 0) - - #Query, by dependent module, for a single module that we know exists - def test_q_dep_mod_Langdon(self): - result = self.fm.query('depmodules', 'Langdon') - self.assertIn('kitty', result) - self.assertEqual(result['kitty']['Dependent Modules'], ['Langdon']) - self.assertEqual(len(result), 1) - def test_q_dep_mod_walrus(self): - result = self.fm.query('depmodules', 'walrus') - self.assertIn('Langdon', result) - self.assertIn('James', result) - self.assertEqual(len(result), 2) - - #Query, by dependent module, for a single module that doesn't exist - def test_q_dep_mod_abc(self): - result = self.fm.query('depmodules', 'ABC_Lic') + r1 = self.fm.query({'data.references.community':'http://www.example.com/'}) + r2 = self.fm.query({'data.references.documentation':'http://www.example.com/'}) + r3 = self.fm.query({'data.references.tracker':'http://www.example.com/'}) + self.assertIn('references', r1['data']) + self.assertIn('references', r2['data']) + self.assertIn('references', r3['data']) + self.assertIn('community', r1['data']['references']) + self.assertIn('documentation', r2['data']['references']) + self.assertIn('tracker', r3['data']['references']) + self.assertEqual(r1['data']['references']['community'], 'http://www.example.com/') + self.assertEqual(r1['data']['references']['documentation'], 'http://www.example.com/') + self.assertEqual(r1['data']['references']['tracker'], 'http://www.example.com/') + + #Query, by references, for a single module that doesn't exist + def test_q_up_docs_fail(self): + r1 = self.fm.query({'data.references.community':'http://www.fakeurl.com/'}) + r2 = self.fm.query({'data.references.documentation':'http://www.fakeurl.com/'}) + r3 = self.fm.query({'data.references.tracker':'http://www.fakeurl.com/'}) + self.assertEqual(len(r1), 0) + self.assertEqual(len(r2), 0) + self.assertEqual(len(r3), 0) + + #Query, by license, for a single module that we know exists + def test_q_license_module_MIT(self): + result = self.fm.query({'data.license.module':['MIT']}) + self.assertIn('license', result['data']) + self.assertIn('module', result['data']['license']) + self.assertEqual(result['data']['license']['module'],['MIT']) + def test_q_license_content_beerware(self): + result = self.fm.query({'data.license.content': {'$in' : ['Beerware']}}) + self.assertIn('license', result['data']) + self.assertIn('content', result['data']['license']) + self.assertEqual(result['data']['license']['content'],['Beerware', 'GPLv2+', 'zlib']) + def test_q_license_content_GPLv2(self): + result = self.fm.query({'data.license.content': {'$in' : ['GPLv2+']}}) + self.assertIn('license', result['data']) + self.assertIn('content', result['data']['license']) + self.assertEqual(result['data']['license']['content'],['Beerware', 'GPLv2+', 'zlib']) + def test_q_license_content_zlib(self): + result = self.fm.query({'data.license.content': {'$in' : ['zlib']}}) + self.assertIn('license', result['data']) + self.assertIn('content', result['data']['license']) + self.assertEqual(result['data']['license']['content'],['Beerware', 'GPLv2+', 'zlib']) + + #Query, by license, for a single module that doesn't exist + def test_q_license_module_fail(self): + result = self.fm.query({'data.license.module':['nonexistent']}) self.assertEqual(len(result), 0) - - #Query, by component RPM, for a single module that we know exists - def test_q_rpms(self): - result = self.fm.query('componentrpms', 'GHI.rpm') - self.assertIn('walrus', result) - self.assertEqual(result['walrus']['Component RPMs'], ['ABC.rpm', 'DEF.rpm', 'GHI.rpm']) - - #Query, by component RPM, for a single module that doesn't exist - def test_q_rpm_nonexistent(self): - result = self.fm.query('componentrpms', 'xyz.rpm') + def test_q_license_content_fail(self): + result = self.fm.query({'data.license.content': {'$in' : ['nonexistent']}}) self.assertEqual(len(result), 0) - - #Test queryOR for 2 modules that we know exist - def test_q_licenses(self): - result = self.fm.queryOR('license', 'Beerware', 'license', 'GPL') - self.assertIn('Langdon', result) - self.assertIn('James', result) - self.assertEqual(len(result), 2) - def test_q_lic_1(self): - result = self.fm.queryOR('name', 'Beerware', 'license', 'GPL') - self.assertIn('James', result) - self.assertEqual(len(result), 1) - - #Test queryOR for 2 modules that don't exist - def test_q_licenses_nonexistent(self): - result = self.fm.queryOR('license', 'non-existent1', 'license', 'non-existent2') - self.assertEqual(len(result), 0) - - #Test queryAND for 1 or 2 modules that we know exist - def test_q_lic_ver_courtney_1(self): - result = self.fm.queryAND('license', 'ABC_Lic', 'version', '13') - self.assertIn('Courtney', result) - self.assertEqual(len(result), 1) - def test_q_lic_ver_courtney_2(self): - result = self.fm.queryAND('version', '13', 'license', 'ABC_Lic') - self.assertIn('Courtney', result) - self.assertEqual(len(result), 1) - def test_q_lic_ver_docs_courtney_1(self): - result = self.fm.queryAND('license', 'ABC_Lic', 'version', '13', - 'upstreamdocs', 'doc') - self.assertIn('Courtney', result) - self.assertEqual(len(result), 1) - def test_q_lic_ver_docs_courtney_2(self): - result = self.fm.queryAND('upstreamdocs', 'doc', 'license', 'ABC_Lic', - 'version', '13') - self.assertIn('Courtney', result) - self.assertEqual(len(result), 1) - - def test_q_lic_ver_langdon(self): - result = self.fm.queryAND('depmodules', 'walrus', 'version', '1') - self.assertIn('Langdon', result) - self.assertEqual(len(result), 1) - - #Test queryAND for 1 or 2 modules that don't exist - def test_q_lic_ver_fail_1(self): - result = self.fm.queryAND('depmodules', 'walrus', 'version', '13') - self.assertEqual(len(result), 0) - - def test_q_lic_ver_fail_2(self): - result = self.fm.queryAND('depmodules', 'walrus', 'version', '3') - self.assertEqual(len(result), 0) - - def test_q_lic_ver_fail_3(self): - result = self.fm.queryAND('license', 'ABC_Lic', 'version', '4') - self.assertEqual(len(result), 0) - - def test_q_lic_ver_fail_4(self): - result = self.fm.queryAND('depmodules', 'ABC_Lic', 'version', '1') - self.assertEqual(len(result), 0) - - def test_q_lic_ver_fail_5(self): - result = self.fm.queryAND('version', '1', 'depmodules', 'ABC_Lic') - self.assertEqual(len(result), 0) - - def test_q_lic_ver_fail_6(self): - result = self.fm.queryAND('version', '1', 'upstreamdocs', 'doc', - 'depmodules', 'ABC_Lic') - self.assertEqual(len(result), 0) - - def test_q_lic_ver_fail_7(self): - result = self.fm.queryAND('version', '1', 'depmodules', 'ABC_Lic', - 'upstreamdocs', 'doc') - self.assertEqual(len(result), 0) - - def test_q_ver_ver_fail_1(self): - result = self.fm.queryAND('version', '4', 'version', '1') - self.assertEqual(len(result), 0) - - #Select entire database via `getTable()` - def test_q_entire_db(self): - result = self.fm.getTable() - databaseContents = '{"Petr": {"Upstream Community": "com", "Dependent Modules": ["kitty"], "License": "ABC_Lic", "URL": "www.Petr.com", "Version": "3", "Upstream Docs": "doc", "Component RPMs": ["ABC.rpm"]}, "Langdon": {"Upstream Community": "com", "Dependent Modules": ["walrus", "kitty"], "License": "Beerware", "URL": "www.Langdon.com", "Version": "1", "Upstream Docs": "doc", "Component RPMs": ["ABC.rpm"]}, "Courtney": {"Upstream Community": "com", "Dependent Modules": ["kitty"], "License": "ABC_Lic", "URL": "www.Courtney.com", "Version": "13", "Upstream Docs": "doc", "Component RPMs": ["ABC.rpm"]}, "kitty": {"Upstream Community": "com", "Dependent Modules": ["Langdon"], "License": "ABC_Lic", "URL": "www.penny.com", "Version": "7", "Upstream Docs": "doc", "Component RPMs": ["ABC.rpm"]}, "Jan": {"Upstream Community": "com", "Dependent Modules": ["kitty"], "License": "ABC_Lic", "URL": "www.Jan.com", "Version": "1", "Upstream Docs": "doc", "Component RPMs": ["ABC.rpm"]}, "James": {"Upstream Community": "com", "Dependent Modules": ["walrus"], "License": "GPL", "URL": "www.James.com", "Version": "4", "Upstream Docs": "doc", "Component RPMs": ["ABC.rpm"]}, "walrus": {"Upstream Community": "com", "Dependent Modules": ["kitty"], "License": "ABC_Lic", "URL": "www.walrus.com", "Version": "1", "Upstream Docs": "doc", "Component RPMs": ["ABC.rpm", "DEF.rpm", "GHI.rpm"]}}' - self.assertEqual(result,json.loads(databaseContents)) From 9c799640d3331f7ba6a4aa4549c9bd48f4a524c4 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: May 11 2016 10:50:12 +0000 Subject: [PATCH 6/11] Get the list of modules from Fedora git repository. Add to database only modules with valid yaml metadata file in modulemd format. Also include the modulemd metadata file in the database so we can send it to fm tool later using the REST API. --- diff --git a/databaseUtilities.py b/databaseUtilities.py index a9eb1d7..c549d61 100644 --- a/databaseUtilities.py +++ b/databaseUtilities.py @@ -1,7 +1,15 @@ import pymongo from pymongo import MongoClient +from bs4 import BeautifulSoup +from urllib import urlopen import json -import yaml +import lxml + +from yaml import load, dump +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper class ModularityDb: @@ -26,28 +34,47 @@ class ModularityDb: self.collection = self.db.ModularityDb self.posts = self.db.posts self.packages = {} - self.__insertData() - - ''' - def __insertData - - @purpose: [Private] insert data into the mongo database - ''' - def __insertData(self): - self.__getData() - self.posts.insert(self.packages) + self.__pullData() ''' - def __getData + def __pullData - @purpose: [Private] get module data from yaml + @purpose: [Private] Pull module data from database ''' - def __getData(self): - with open("metadata.yaml","r") as stream: + def __pullData(self): + r = urlopen('https://admin.stg.fedoraproject.org/pkgdb/api/packages?namespace=modules') + soup = BeautifulSoup(r,"lxml") + text = soup.getText() + json_data = json.loads(text) + + self.packages = [] + + # Check all the packages in the 'modules' namespace and include + # only packages with the valid modulemd yaml file. Also include the + # modulemd file in the database, so we can later serve it using the + # REST API. + for package in json_data['packages']: + url = 'http://pkgs.stg.fedoraproject.org/cgit/modules/{0}.git/plain/{0}.yaml'.format(package['name']) + r = urlopen(url) + data = r.read() + + # Try to parse the modulemd.yaml try: - self.packages = yaml.load(stream) - except yaml.YAMLError as exc: - print(exc) + obj = load(data, Loader=Loader) + except: + print("{0} does not have valid {0}.yaml file".format(package['name'])) + continue + + # Check if it modulemd document. + if not isinstance(obj, dict) or not "document" in obj or obj["document"] != "modulemd": + print("{0} does not have valid {0}.yaml file".format(package['name'])) + continue + + print("{0} added to database".format(package['name'])) + # modulemd.yaml exists and it is yaml, so we can store it. + package["modulemd"] = data + self.packages.append(package) + self.posts.insert(self.packages) ''' def printDatabase @@ -88,17 +115,4 @@ class ModularityDb: result = [] for post in self.posts.find(query_dict): result.append(post) - if not result: - result.append(list(self.collection.find(query_dict))) - return result - else: - return result - - ''' - def cleanup - - @purpose: Clear the database - ''' - def cleanup(self): - c = pymongo.Connection() - c.drop_database('ModularityDb') + return result diff --git a/demo.py b/demo.py index 200dfa2..fdab030 100644 --- a/demo.py +++ b/demo.py @@ -7,15 +7,10 @@ myDb = ModularityDb() myDb.printDatabase() #Test Queries -q1 = {'document': 'modulemd'} -q2 = {'version':2000} -q3 = {'version': 0} -q4 = {'data.components.rpms.dependencies': True} +q1 = {"name":"testmodule"} +q2 = {"name":"doesnotexist"} +q3 = {"koschei_monitor" : True} print "q1:\n", myDb.query(q1) print "q2:\n",myDb.query(q2) #empty result! print "q3:\n", myDb.query(q3) -print "q4:\n", myDb.query(q4) - -#Clear database -myDb.cleanup() diff --git a/metadata.yaml b/metadata.yaml deleted file mode 100644 index 5e101f4..0000000 --- a/metadata.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# Document type identifier -document: modulemd -# Module metadata format version -version: 0 -data: - # Module name, required - # NOTE: Module names might be structured differently later to include the - # module's vendor ecosystem, e.g. org.fedoraproject.foo in this case. - # This would also affect the dependency sections defined furhter below. - name: foo - # NOTE: Module versioning is currently being investigated - # http://taiga.fedorainfracloud.org/project/modularity/us/175 - # Module version, required - # Typically the same as the version of the main component, where applicable - version: 1.23 - # Module release, required - # Version of the module itself - release: 4 - # A short summary describing the module, required - summary: An example module - # A verbose description of the module, required - description: > - A module for the demonstration of the metadata format. Also, - the obligatory lorem ipsum dolor sit amet goes right here. - # Module and content licenses in the Fedora license identifier format, required - license: - # Module license, required - # This list covers licenses used for the module metadata, - # SPEC files or extra patches - module: - - MIT - # Content license, optional - # A list of licenses used by the packages in the module. - # This should be populated by build tools. - content: - - Beerware - - GPLv2+ - - zlib - # Extensible metadata block - # http://taiga.fedorainfracloud.org/project/modularity/us/135 - # A dictionary of user-defined keys and values. May be null. - # Optional. Defaults to null. - xmd: ~ - # Module dependencies, if any. Optional. - # NOTE: Module dependencies are currently being investigated - # http://taiga.fedorainfracloud.org/project/modularity/us/170 - # TODO: Provides, conflicts, obsoletes, recommends, etc. - # TODO: Support for comparison operators - dependencies: - # Build dependencies of this module, optional - # Keys are module names, values are the minimum required versions - # These modules define the buildroot for this module - buildrequires: - core: 23 - c-build: 6.0 - # Run-time dependencies of this module, optional - # Keys are module names, values are the minimum required versions - requires: - core: 23 - # References to external resources, typically upstream, optional - references: - # Upstream community website, if it exists, optional - community: http://www.example.com/ - # Upstream documentation, if it exists, optional - documentation: http://www.example.com/ - # Upstream bug tracker, if it exists, optional - tracker: http://www.example.com/ - # Functional components of the module, optional - components: - # RPM content of the module, optional - # NOTE: This is currently being investigated - # TODO: Package priority build order support (chainbuild) - # TODO: Binary package filtering (inclusive or exclusive) support - # TODO: Component tags: - # http://taiga.fedorainfracloud.org/project/modularity/us/186 - # http://taiga.fedorainfracloud.org/project/modularity/us/187 - # http://taiga.fedorainfracloud.org/project/modularity/us/188 - # TODO: Define architectures to build for - rpms: - # Should this module include dependencies of its RPM packages - # that are not provided by any of the modules it depends on? - # Optional, defaults to True - dependencies: True - # Should this module include all subpackages generated from the - # source packages? True if yes, False if only the main packages - # should be included. - # Optional, defaults to True - fulltree: True - # RPM-based packages of this module. - # Keys are the VCS/SRPM names, values dictionaries holding - # additional information where required, or null. - # Optional - packages: - bar: - # Use this repository if it's different from - # the build system configuration. - # Optional. - repository: https://pagure.io/bar.git - # Use this lookaside cache if it's different - # from the build system configuration. - # Optional. - cache: https://example.com/cache - # Use this specific commit ID for the build. - # If no commit ID is given, the latest master - # commit will be used. - # Optional. - commit: 26ca0c0 - # baz has no extra options - baz: ~ - xxx: - # xxx is only available on the listed architectures. - # Optional, defaults to all available arches. - arches: [ i686, x86_64 ] - # a list of architectures with multilib - # installs, i.e. both i686 and x86_64 - # versions will be installed on x86_64. - # Optional, defaults to no multilib. - multilib: [ x86_64 ] - xyz: ~ From 70333277e79c48b02b04566b012d632456dfd73e Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: May 11 2016 11:00:02 +0000 Subject: [PATCH 7/11] Answer with proper data from database in fm.py CGI script --- diff --git a/fm.py b/fm.py index d1157e5..4f3850b 100644 --- a/fm.py +++ b/fm.py @@ -8,6 +8,8 @@ import cgi # Show tracebacks ... import cgitb +import json + # For later when we don't want to show the user... # cgitb.enable(display=0, logdir="/path/to/logdir") @@ -25,20 +27,14 @@ if not args['path']: path = args['path'][0] myDb = ModularityDb() -import json if path == '/list': - ret = [] - ret.append(myDb.getDatabase()) - print json.dumps(ret) + print json.dumps(myDb.getDatabase()) elif path.startswith("/info/"): name = path[len("/info/"):] - ret = myDb.getDatabase() - if bool(ret) == False: - print json.dumps({'error' : 'not found'}) - sys.exit(1) - print json.dumps([ret]) + q1 = {"name": name} + print json.dumps(myDb.query(q1)) else: print json.dumps({'error' : 'bad arg'}) From d845e5c754337fbbb4fe4300a12ea08f6e26b135 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: May 11 2016 11:05:50 +0000 Subject: [PATCH 8/11] Remove debug output --- diff --git a/databaseUtilities.py b/databaseUtilities.py index c549d61..5521b90 100644 --- a/databaseUtilities.py +++ b/databaseUtilities.py @@ -62,15 +62,12 @@ class ModularityDb: try: obj = load(data, Loader=Loader) except: - print("{0} does not have valid {0}.yaml file".format(package['name'])) continue # Check if it modulemd document. if not isinstance(obj, dict) or not "document" in obj or obj["document"] != "modulemd": - print("{0} does not have valid {0}.yaml file".format(package['name'])) continue - print("{0} added to database".format(package['name'])) # modulemd.yaml exists and it is yaml, so we can store it. package["modulemd"] = data self.packages.append(package) From cfe6e246f6144e9898a3f4506092440801229522 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: May 11 2016 11:25:19 +0000 Subject: [PATCH 9/11] Use dumps from bson.json_util to support MongoDB's ObjectID dump to JSON --- diff --git a/databaseUtilities.py b/databaseUtilities.py index 5521b90..502c0c0 100644 --- a/databaseUtilities.py +++ b/databaseUtilities.py @@ -71,7 +71,7 @@ class ModularityDb: # modulemd.yaml exists and it is yaml, so we can store it. package["modulemd"] = data self.packages.append(package) - self.posts.insert(self.packages) + self.posts.insert(dict(self.packages)) ''' def printDatabase diff --git a/fm.py b/fm.py index 4f3850b..a7f226a 100644 --- a/fm.py +++ b/fm.py @@ -9,6 +9,7 @@ import cgi import cgitb import json +from bson.json_util import dumps # For later when we don't want to show the user... # cgitb.enable(display=0, logdir="/path/to/logdir") @@ -29,12 +30,12 @@ path = args['path'][0] myDb = ModularityDb() if path == '/list': - print json.dumps(myDb.getDatabase()) + print dumps(myDb.getDatabase()) elif path.startswith("/info/"): name = path[len("/info/"):] q1 = {"name": name} - print json.dumps(myDb.query(q1)) + print dumps(myDb.query(q1)) else: print json.dumps({'error' : 'bad arg'}) From 3588fbff2a1f8235a629488263c52c2cafecfd10 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: May 11 2016 11:26:42 +0000 Subject: [PATCH 10/11] Remove wrong dict() type-cast... --- diff --git a/databaseUtilities.py b/databaseUtilities.py index 502c0c0..5521b90 100644 --- a/databaseUtilities.py +++ b/databaseUtilities.py @@ -71,7 +71,7 @@ class ModularityDb: # modulemd.yaml exists and it is yaml, so we can store it. package["modulemd"] = data self.packages.append(package) - self.posts.insert(dict(self.packages)) + self.posts.insert(self.packages) ''' def printDatabase From db63480586d6edea40a296107759cb280fa6d5a9 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: May 11 2016 11:32:56 +0000 Subject: [PATCH 11/11] Cleanup database after the request for now, this has to be rewritten so we are not adding data to database on every request. --- diff --git a/databaseUtilities.py b/databaseUtilities.py index 5521b90..3d395c7 100644 --- a/databaseUtilities.py +++ b/databaseUtilities.py @@ -113,3 +113,11 @@ class ModularityDb: for post in self.posts.find(query_dict): result.append(post) return result + + ''' + def cleanup + + @purpose: Clear the database + ''' + def cleanup(self): + self.client.drop_database('ModularityDb') diff --git a/fm.py b/fm.py index a7f226a..f20e97a 100644 --- a/fm.py +++ b/fm.py @@ -26,7 +26,6 @@ if not args['path']: print json.dumps({'error' : 'bad path'}) sys.exit(1) path = args['path'][0] - myDb = ModularityDb() if path == '/list': @@ -39,4 +38,5 @@ elif path.startswith("/info/"): else: print json.dumps({'error' : 'bad arg'}) - sys.exit(1) + +myDb.cleanup()