From 887ece07bad8a32a1eca6794ade079b9644fd976 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:57 +0000 Subject: [PATCH 1/27] Initial commit of a Flask API. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 6be2a26..12da9b5 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,68 +1,16 @@ -"""Kiskadee API.""" -from flask import Flask, jsonify -from flask import request -from flask_cors import CORS - +from flask import Flask from kiskadee.database import Database -from kiskadee.model import Package, Fetcher, Version, Analysis -from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ - AnalysisSchema +from kiskadee.model import Package, Fetcher, Version kiskadee = Flask(__name__) - -CORS(kiskadee) +db_session = Database().session -@kiskadee.route('/fetchers') +@kiskadee.route('/') def index(): - """Get the list of available fetchers.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - fetchers = db_session.query(Fetcher).all() - fetcher_schema = FetcherSchema(many=True) - result = fetcher_schema.dump(fetchers) - return jsonify({'fetchers': result.data}) - - -@kiskadee.route('/packages') -def packages(): - """Get the list of analyzed packages.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - packages = db_session.query(Package).all() - package_schema = PackageSchema(many=True) - result = package_schema.dump(packages) - return jsonify({'packages': result.data}) - - -@kiskadee.route('/analysis///') -def package_analysis(pkg_name, version): - """Get the a analysis of some package version.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - package = ( - db_session.query(Package) - .filter(Package.name == pkg_name).first().id - ) - version = ( - db_session.query(Version) - .filter(Version.package_id == package).first().id - ) - analysis = ( - db_session.query(Analysis) - .filter(Analysis.version_id == version).first() - ) - - analysis_schema = AnalysisSchema() - result = analysis_schema.dump(analysis) - return jsonify({'analysis': result.data}) - + return db_session.query(Package).first().name -def kiskadee_db_session(): - """Return a kiskadee database session.""" - return Database().session +if __name__ == '__main__': + kiskadee.run(debug=True) -def main(): - """Initialize the kiskadee API.""" - kiskadee.run('0.0.0.0') diff --git a/requirements.txt b/requirements.txt index f1af6c5..953963a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,11 +12,4 @@ packaging pyyaml flake8 pydocstyle -coverage -nose flask -Flask-Restless -marshmallow -flask-cors -coverage -nose From 8efd73098f7477babad1cb5ccdab747ad41dcf1a Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:57 +0000 Subject: [PATCH 2/27] Add endpoint to get analyzed packages. - Add endpoint to get the available fetchers. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 12da9b5..da0448a 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,16 +1,30 @@ -from flask import Flask +import json +from flask import Flask, jsonify +from flask import request + from kiskadee.database import Database from kiskadee.model import Package, Fetcher, Version +from kiskadee.api.serializers import PackageSchema, FetcherSchema kiskadee = Flask(__name__) db_session = Database().session -@kiskadee.route('/') +@kiskadee.route('/fetchers') def index(): - return db_session.query(Package).first().name + if request.method == 'GET': + fetchers = db_session.query(Fetcher).all() + fetcher_schema = FetcherSchema(many=True) + result = fetcher_schema.dump(fetchers) + return jsonify({'fetcher': result.data}) +@kiskadee.route('/packages') +def packages(): + if request.method == 'GET': + packages = db_session.query(Package).all() + package_schema = PackageSchema(many=True) + result = package_schema.dump(packages) + return jsonify({'packages': result.data}) if __name__ == '__main__': kiskadee.run(debug=True) - diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 3b04abc..9bb9072 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -1,46 +1,22 @@ -"""Provide objects to serialize the kiskadee models.""" - -from marshmallow import Schema, fields -from kiskadee.model import Package, Fetcher, Analysis - +from marshmallow import Schema, fields, ValidationError, pre_load class FetcherSchema(Schema): - """Provide a serializer to the Fetcher model.""" - id = fields.Int() name = fields.Str() target = fields.Str() description = fields.Str() def make_object(self, data): - """Serialize a Fetcher object.""" print('MAKING OBJECT FROM', data) return Fetcher(**data) - class PackageSchema(Schema): - """Provide a serializer to the Package model.""" - id = fields.Int() name = fields.Str() target = fields.Str() fetcher_id = fields.Nested(FetcherSchema) def make_object(self, data): - """Serialize a Package object.""" print('MAKING OBJECT FROM', data) return Package(**data) - -class AnalysisSchema(Schema): - """Provide a serializer to the Analysis model.""" - - id = fields.Int() - version_id = fields.Int() - analyzer_id = fields.Int() - raw = fields.Str() - - def make_object(self, data): - """Serialize a Analysis object.""" - print('MAKING OBJECT FROM', data) - return Analysis(**data) diff --git a/requirements.txt b/requirements.txt index 953963a..fbff5eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,5 @@ pyyaml flake8 pydocstyle flask +Flask-Restless +marshmallow From 17d41fb23a0e2cdfe2adc98a7ef9e587acae555b Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:57 +0000 Subject: [PATCH 3/27] Add endpoint to get a package analysis. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index da0448a..c609bcd 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -3,8 +3,9 @@ from flask import Flask, jsonify from flask import request from kiskadee.database import Database -from kiskadee.model import Package, Fetcher, Version -from kiskadee.api.serializers import PackageSchema, FetcherSchema +from kiskadee.model import Package, Fetcher, Version, Analysis +from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ + AnalysisSchema kiskadee = Flask(__name__) db_session = Database().session @@ -26,5 +27,19 @@ def packages(): result = package_schema.dump(packages) return jsonify({'packages': result.data}) +@kiskadee.route('/analysis///') +def package_analysis(pkg_name, version): + if request.method == 'GET': + package = db_session.query(Package)\ + .filter(Package.name == pkg_name).first().id + version = db_session.query(Version)\ + .filter(Version.package_id == package).first().id + analysis = db_session.query(Analysis)\ + .filter(Analysis.version_id == version).first() + + analysis_schema = AnalysisSchema() + result = analysis_schema.dump(analysis) + return jsonify({'analysis': result.data}) + if __name__ == '__main__': kiskadee.run(debug=True) diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 9bb9072..34f1866 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -20,3 +20,12 @@ class PackageSchema(Schema): print('MAKING OBJECT FROM', data) return Package(**data) +class AnalysisSchema(Schema): + id = fields.Int() + version_id = fields.Int() + analyzer_id = fields.Int() + raw = fields.Str() + + def make_object(self, data): + print('MAKING OBJECT FROM', data) + return Analysis(**data) From f71d87f5b00125becd945b745cb1933165c4584b Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:57 +0000 Subject: [PATCH 4/27] Update Jenkinsfile. --- diff --git a/Jenkinsfile b/Jenkinsfile index f20c554..86c41d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { } stage('build-docker-images') { steps { - sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' + sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' sh '(cd util/dockerfiles/flawfinder && docker build . -t flawfinder)' } } From 3b412928bf5acdc8c9158bb0306395eae2b72a94 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 5/27] Add cross origin support --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index c609bcd..4dc1889 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,6 +1,7 @@ import json from flask import Flask, jsonify from flask import request +from flask_cors import CORS, cross_origin from kiskadee.database import Database from kiskadee.model import Package, Fetcher, Version, Analysis @@ -9,7 +10,7 @@ from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ kiskadee = Flask(__name__) db_session = Database().session - +CORS(kiskadee) @kiskadee.route('/fetchers') def index(): @@ -27,6 +28,7 @@ def packages(): result = package_schema.dump(packages) return jsonify({'packages': result.data}) + @kiskadee.route('/analysis///') def package_analysis(pkg_name, version): if request.method == 'GET': @@ -42,4 +44,4 @@ def package_analysis(pkg_name, version): return jsonify({'analysis': result.data}) if __name__ == '__main__': - kiskadee.run(debug=True) + kiskadee.run('0.0.0.0') diff --git a/requirements.txt b/requirements.txt index fbff5eb..1ba4deb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,4 @@ pydocstyle flask Flask-Restless marshmallow +flask-cors From 14d5402be84f5413726940be085520bf21bd080f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 6/27] use jenkins user to run commands. --- diff --git a/Jenkinsfile b/Jenkinsfile index 86c41d3..c13f226 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,37 +1,26 @@ pipeline { agent any + environment { + USER = "jenkins" + } + stages { stage('Build') { steps { - sh 'virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && pip install -e .' + sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' + sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} pip install -e .' } } stage('build-docker-images') { steps { - sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' - sh '(cd util/dockerfiles/flawfinder && docker build . -t flawfinder)' + sh '(sudo -H -u ${USER} cd util/dockerfiles/cppcheck && sudo -H -u ${USER} docker build . -t cppcheck)' + sh '(sudo -H -u ${USER} cd util/dockerfiles/flawfinder && sudo -H -u ${USER} docker build . -t flawfinder)' } } stage('Test') { steps { - sh "chmod u+x run_tests_and_coverage.sh" - sh "source bin/activate && ./run_tests_and_coverage.sh" - } - - post { - success { - // publish html - publishHTML target: [ - allowMissing: false, - alwaysLinkToLastBuild: false, - keepAll: true, - reportDir: 'htmlcov', - reportFiles: 'index.html', - reportName: 'coverage report' - ] - } + sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} python setup.py test' } } } From 5fbbe6e28d7bdc6e717103045e2314e7212e2b44 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 7/27] Change repo owner. --- diff --git a/Jenkinsfile b/Jenkinsfile index c13f226..3b6f99b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,6 +6,11 @@ pipeline { } stages { + stage('change-repo-owner') { + steps { + sh 'chown -R ${USER}.${USER} .' + } + } stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' From 71805dba801cafd6b3454dbeec217f8e78572610 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 8/27] Activate the virtualenv properly. --- diff --git a/Jenkinsfile b/Jenkinsfile index 3b6f99b..101fd9f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} pip install -e .' + sh 'source bin/activate && sudo -H -u ${USER} pip install -e .' } } stage('build-docker-images') { @@ -25,7 +25,7 @@ pipeline { } stage('Test') { steps { - sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} python setup.py test' + sh 'source bin/activate && sudo -H -u ${USER} python setup.py test' } } } From 051a80313c6edafeeabcfbf645f915b4588012d2 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 9/27] Source the virtualenv functions. --- diff --git a/Jenkinsfile b/Jenkinsfile index 101fd9f..e7ca1e5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && sudo -H -u ${USER} pip install -e .' + sudo -H -u ${USER} sh -c 'source bin/activate && pip install -e .' } } stage('build-docker-images') { @@ -25,7 +25,7 @@ pipeline { } stage('Test') { steps { - sh 'source bin/activate && sudo -H -u ${USER} python setup.py test' + sudo -H -u ${USER} sh -c 'source bin/activate && python setup.py test' } } } From a3bd002461374e998b68daab20919bc4fb0d3154 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 10/27] Fix typo. --- diff --git a/Jenkinsfile b/Jenkinsfile index e7ca1e5..0224166 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sudo -H -u ${USER} sh -c 'source bin/activate && pip install -e .' + sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .' } } stage('build-docker-images') { @@ -25,7 +25,7 @@ pipeline { } stage('Test') { steps { - sudo -H -u ${USER} sh -c 'source bin/activate && python setup.py test' + sudo -H -u jenkins sh -c 'source bin/activate && python setup.py test' } } } From 48bf5963c519093c894cec86b69d49f6fccea67e Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 11/27] User sh to run commands. --- diff --git a/Jenkinsfile b/Jenkinsfile index 0224166..76b9d45 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .' + sh "sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .'" } } stage('build-docker-images') { @@ -25,7 +25,7 @@ pipeline { } stage('Test') { steps { - sudo -H -u jenkins sh -c 'source bin/activate && python setup.py test' + sh "sudo -H -u jenkins sh -c 'source bin/activate && python setup.py test'" } } } From f519a8298daef061be49be12eba38ee0a2c26dff Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 12/27] Use default jenkins user. --- diff --git a/Jenkinsfile b/Jenkinsfile index 76b9d45..25bd3c3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,10 +1,6 @@ pipeline { agent any - environment { - USER = "jenkins" - } - stages { stage('change-repo-owner') { steps { @@ -13,19 +9,20 @@ pipeline { } stage('Build') { steps { - sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sh "sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .'" + sh 'virtualenv -p /usr/bin/python3 .' + sh 'source bin/activate && pip install -e .' } } stage('build-docker-images') { steps { - sh '(sudo -H -u ${USER} cd util/dockerfiles/cppcheck && sudo -H -u ${USER} docker build . -t cppcheck)' - sh '(sudo -H -u ${USER} cd util/dockerfiles/flawfinder && sudo -H -u ${USER} docker build . -t flawfinder)' + sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' + sh '(cd util/dockerfiles/flawfinder && docker build . -t flawfinder)' } } stage('Test') { steps { - sh "sudo -H -u jenkins sh -c 'source bin/activate && python setup.py test'" + sh "echo $UID && echo $USER" + sh "source bin/activate && python setup.py test" } } } From fcfe74d5d87c4b95702d2c05387969761f35f59b Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 13/27] Fix typo. --- diff --git a/Jenkinsfile b/Jenkinsfile index 25bd3c3..2fe8af0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,15 +2,10 @@ pipeline { agent any stages { - stage('change-repo-owner') { - steps { - sh 'chown -R ${USER}.${USER} .' - } - } stage('Build') { steps { sh 'virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && pip install -e .' + sh 'source bin/activate && pip install -e .' } } stage('build-docker-images') { From 3871851289e47faf956ac7c88bb5901680c6e229 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 14/27] Remove unused code from Jenkinsfile. --- diff --git a/Jenkinsfile b/Jenkinsfile index 2fe8af0..a311405 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,6 @@ pipeline { } stage('Test') { steps { - sh "echo $UID && echo $USER" sh "source bin/activate && python setup.py test" } } From efb7154e7270a827a4af802e7c032292ec7404c5 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:58 +0000 Subject: [PATCH 15/27] Fix tests --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 4dc1889..4a96917 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,7 +1,7 @@ -import json +"""Kiskadee API.""" from flask import Flask, jsonify from flask import request -from flask_cors import CORS, cross_origin +from flask_cors import CORS from kiskadee.database import Database from kiskadee.model import Package, Fetcher, Version, Analysis @@ -12,16 +12,20 @@ kiskadee = Flask(__name__) db_session = Database().session CORS(kiskadee) + @kiskadee.route('/fetchers') def index(): + """Get the list of available fetchers.""" if request.method == 'GET': fetchers = db_session.query(Fetcher).all() fetcher_schema = FetcherSchema(many=True) result = fetcher_schema.dump(fetchers) return jsonify({'fetcher': result.data}) + @kiskadee.route('/packages') def packages(): + """Get the list of analyzed packages.""" if request.method == 'GET': packages = db_session.query(Package).all() package_schema = PackageSchema(many=True) @@ -31,17 +35,23 @@ def packages(): @kiskadee.route('/analysis///') def package_analysis(pkg_name, version): + """Get the a analysis of some package version.""" if request.method == 'GET': package = db_session.query(Package)\ .filter(Package.name == pkg_name).first().id - version = db_session.query(Version)\ + version = ( + db_session.query(Version) .filter(Version.package_id == package).first().id - analysis = db_session.query(Analysis)\ + ) + analysis = ( + db_session.query(Analysis) .filter(Analysis.version_id == version).first() + ) analysis_schema = AnalysisSchema() result = analysis_schema.dump(analysis) return jsonify({'analysis': result.data}) + if __name__ == '__main__': kiskadee.run('0.0.0.0') diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 34f1866..3b04abc 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -1,31 +1,46 @@ -from marshmallow import Schema, fields, ValidationError, pre_load +"""Provide objects to serialize the kiskadee models.""" + +from marshmallow import Schema, fields +from kiskadee.model import Package, Fetcher, Analysis + class FetcherSchema(Schema): + """Provide a serializer to the Fetcher model.""" + id = fields.Int() name = fields.Str() target = fields.Str() description = fields.Str() def make_object(self, data): + """Serialize a Fetcher object.""" print('MAKING OBJECT FROM', data) return Fetcher(**data) + class PackageSchema(Schema): + """Provide a serializer to the Package model.""" + id = fields.Int() name = fields.Str() target = fields.Str() fetcher_id = fields.Nested(FetcherSchema) def make_object(self, data): + """Serialize a Package object.""" print('MAKING OBJECT FROM', data) return Package(**data) + class AnalysisSchema(Schema): + """Provide a serializer to the Analysis model.""" + id = fields.Int() version_id = fields.Int() analyzer_id = fields.Int() raw = fields.Str() def make_object(self, data): + """Serialize a Analysis object.""" print('MAKING OBJECT FROM', data) return Analysis(**data) From 4a626729df6e0ab6aaa7d2f591623bbb94b11a6e Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 16/27] Add test coverage. --- diff --git a/Jenkinsfile b/Jenkinsfile index a311405..ce64eb7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,7 @@ pipeline { } stage('Test') { steps { - sh "source bin/activate && python setup.py test" + sh "source bin/activate && python kiskadee_coverage.py" } } } diff --git a/kiskadee_coverage.py b/kiskadee_coverage.py index b8014df..80894ca 100644 --- a/kiskadee_coverage.py +++ b/kiskadee_coverage.py @@ -6,8 +6,7 @@ sources = [ 'kiskadee.queue', 'kiskadee.runner', 'kiskadee.model', - 'kiskadee.util', - 'kiskadee.api.app' + 'kiskadee.util' ] cov = Coverage(source=sources, omit="lib/*") diff --git a/requirements.txt b/requirements.txt index 1ba4deb..312cf76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,5 @@ flask Flask-Restless marshmallow flask-cors +coverage +nose From 98e5cc0d4c40ec6fa240b13f755eb1be8ae1f249 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 17/27] Publish the html report. --- diff --git a/Jenkinsfile b/Jenkinsfile index ce64eb7..a7d4b9a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,6 +18,20 @@ pipeline { steps { sh "source bin/activate && python kiskadee_coverage.py" } + + post { + success { + // publish html + publishHTML target: [ + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'covhtml', + reportFiles: 'index.html', + reportName: 'coverage report' + ] + } + } } } } From 19a82afd253b42d5fd2599550cfe6450b124fe25 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 18/27] Initial test to kiskadee api. --- diff --git a/kiskadee/tests/test_api.py b/kiskadee/tests/test_api.py index fa04b3c..e8fe09f 100644 --- a/kiskadee/tests/test_api.py +++ b/kiskadee/tests/test_api.py @@ -1,48 +1,29 @@ import json +from kiskadee.api.app import kiskadee import unittest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker import kiskadee.model as model -import kiskadee -from kiskadee.api.app import kiskadee as kiskadee_api -import kiskadee.api.app class ApiTestCase(unittest.TestCase): def setUp(self): - kiskadee_api.testing = True + kiskadee.testing = True + self.app = kiskadee.test_client() self.engine = create_engine('sqlite:///:memory:') Session = sessionmaker(bind=self.engine) self.session = Session() - self.app = kiskadee_api.test_client() model.Base.metadata.create_all(self.engine) model.create_analyzers(self.session) - fetcher = model.Fetcher( + self.fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' - ) - self.session.add(fetcher) - self.session.commit() + ) def test_get_fetchers(self): - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session - response = self.app.get("/fetchers") - self.assertIn("fetchers", json.loads(response.data.decode("utf-8"))) - - def test_get_activated_fetcher(self): - - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session response = self.app.get("/fetchers") - response_as_json = json.loads(response.data.decode("utf-8")) - fetcher_name = response_as_json["fetchers"][0]["name"] - self.assertEqual("kiskadee-fetcher", fetcher_name) + self.assertIn("fetcher", json.loads(response.data.decode("utf-8"))) if __name__ == '__main__': diff --git a/kiskadee_coverage.py b/kiskadee_coverage.py index 80894ca..b8014df 100644 --- a/kiskadee_coverage.py +++ b/kiskadee_coverage.py @@ -6,7 +6,8 @@ sources = [ 'kiskadee.queue', 'kiskadee.runner', 'kiskadee.model', - 'kiskadee.util' + 'kiskadee.util', + 'kiskadee.api.app' ] cov = Coverage(source=sources, omit="lib/*") From 89b276daec61fafaf1831c8d12955eced8f0b1fd Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 19/27] Save analysis as JSON. - related to #40 - This commit generated the bug related with the issue #44. We need to fix it, to release the 0.3 version. --- diff --git a/kiskadee/converter.py b/kiskadee/converter.py index 12ae044..dd38571 100644 --- a/kiskadee/converter.py +++ b/kiskadee/converter.py @@ -7,8 +7,6 @@ from importlib import import_module import shutil import tempfile import os -import json - from firehose.model import Analysis, to_json @@ -40,7 +38,7 @@ def to_firehose(bytes_input, analyzer): analysis_as_json = to_json(Analysis.from_xml(f)) shutil.rmtree(tempdir) - return json.dumps(analysis_as_json) + return analysis_as_json def import_firehose_parser(parser): From 554cdaf6bb69b857bdce23c14862b83309acd734 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 20/27] Generate the coverage directly from setup.py. --- diff --git a/Jenkinsfile b/Jenkinsfile index a7d4b9a..f20c554 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,8 @@ pipeline { } stage('Test') { steps { - sh "source bin/activate && python kiskadee_coverage.py" + sh "chmod u+x run_tests_and_coverage.sh" + sh "source bin/activate && ./run_tests_and_coverage.sh" } post { @@ -26,7 +27,7 @@ pipeline { allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'covhtml', + reportDir: 'htmlcov', reportFiles: 'index.html', reportName: 'coverage report' ] diff --git a/run_tests_and_coverage.sh b/run_tests_and_coverage.sh index 2634599..ef480cc 100755 --- a/run_tests_and_coverage.sh +++ b/run_tests_and_coverage.sh @@ -1,3 +1,3 @@ #!/bin/bash -coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*" ./setup.py test +coverage run --omit="lib/*","setup.py","kiskadee/tests/*" ./setup.py test coverage html From 46cc7532960cd1eb7e58a28a05aa59d38f035ecc Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 21/27] Ignore .eggs file when run the coverage. --- diff --git a/run_tests_and_coverage.sh b/run_tests_and_coverage.sh index ef480cc..2634599 100755 --- a/run_tests_and_coverage.sh +++ b/run_tests_and_coverage.sh @@ -1,3 +1,3 @@ #!/bin/bash -coverage run --omit="lib/*","setup.py","kiskadee/tests/*" ./setup.py test +coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*" ./setup.py test coverage html From 90af9d753d3528bf8f8874098ca8cbbc9c6cd669 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 22/27] Add an architecture section to the docs. --- diff --git a/doc/architecture.rst b/doc/architecture.rst index 1dbd1c0..8e22ca5 100644 --- a/doc/architecture.rst +++ b/doc/architecture.rst @@ -27,3 +27,4 @@ and this was a scope decision made by the kiskadee community. .. *Figure One: Kiskadee architecture.* + From acf3be6b958bb106e46e8019810bdf1db4a41864 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 23/27] Make api tests not depend of a postgresql database --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 4a96917..6be2a26 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -9,7 +9,7 @@ from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ AnalysisSchema kiskadee = Flask(__name__) -db_session = Database().session + CORS(kiskadee) @@ -17,16 +17,18 @@ CORS(kiskadee) def index(): """Get the list of available fetchers.""" if request.method == 'GET': + db_session = kiskadee_db_session() fetchers = db_session.query(Fetcher).all() fetcher_schema = FetcherSchema(many=True) result = fetcher_schema.dump(fetchers) - return jsonify({'fetcher': result.data}) + return jsonify({'fetchers': result.data}) @kiskadee.route('/packages') def packages(): """Get the list of analyzed packages.""" if request.method == 'GET': + db_session = kiskadee_db_session() packages = db_session.query(Package).all() package_schema = PackageSchema(many=True) result = package_schema.dump(packages) @@ -37,8 +39,11 @@ def packages(): def package_analysis(pkg_name, version): """Get the a analysis of some package version.""" if request.method == 'GET': - package = db_session.query(Package)\ + db_session = kiskadee_db_session() + package = ( + db_session.query(Package) .filter(Package.name == pkg_name).first().id + ) version = ( db_session.query(Version) .filter(Version.package_id == package).first().id @@ -53,5 +58,11 @@ def package_analysis(pkg_name, version): return jsonify({'analysis': result.data}) -if __name__ == '__main__': +def kiskadee_db_session(): + """Return a kiskadee database session.""" + return Database().session + + +def main(): + """Initialize the kiskadee API.""" kiskadee.run('0.0.0.0') diff --git a/kiskadee/converter.py b/kiskadee/converter.py index dd38571..12ae044 100644 --- a/kiskadee/converter.py +++ b/kiskadee/converter.py @@ -7,6 +7,8 @@ from importlib import import_module import shutil import tempfile import os +import json + from firehose.model import Analysis, to_json @@ -38,7 +40,7 @@ def to_firehose(bytes_input, analyzer): analysis_as_json = to_json(Analysis.from_xml(f)) shutil.rmtree(tempdir) - return analysis_as_json + return json.dumps(analysis_as_json) def import_firehose_parser(parser): diff --git a/kiskadee/runner.py b/kiskadee/runner.py index f3b6b80..e115353 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -131,10 +131,10 @@ class Runner: ) ) uncompressed_source_path = tempfile.mkdtemp() - try: - shutil.unpack_archive( - compressed_source, - uncompressed_source_path + shutil.unpack_archive(compressed_source, uncompressed_source_path) + kiskadee.logger.debug( + 'ANALYSIS: Unpacking {} source in {} path' + .format(package['name'], uncompressed_source_path) ) kiskadee.logger.debug( 'ANALYSIS: Unpacking {} source in {} path' diff --git a/kiskadee/tests/test_api.py b/kiskadee/tests/test_api.py index e8fe09f..fa04b3c 100644 --- a/kiskadee/tests/test_api.py +++ b/kiskadee/tests/test_api.py @@ -1,29 +1,48 @@ import json -from kiskadee.api.app import kiskadee import unittest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker import kiskadee.model as model +import kiskadee +from kiskadee.api.app import kiskadee as kiskadee_api +import kiskadee.api.app class ApiTestCase(unittest.TestCase): def setUp(self): - kiskadee.testing = True - self.app = kiskadee.test_client() + kiskadee_api.testing = True self.engine = create_engine('sqlite:///:memory:') Session = sessionmaker(bind=self.engine) self.session = Session() + self.app = kiskadee_api.test_client() model.Base.metadata.create_all(self.engine) model.create_analyzers(self.session) - self.fetcher = model.Fetcher( + fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' - ) + ) + self.session.add(fetcher) + self.session.commit() def test_get_fetchers(self): + def mock_kiskadee_db_session(): + return self.session + + kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session + response = self.app.get("/fetchers") + self.assertIn("fetchers", json.loads(response.data.decode("utf-8"))) + + def test_get_activated_fetcher(self): + + def mock_kiskadee_db_session(): + return self.session + + kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session response = self.app.get("/fetchers") - self.assertIn("fetcher", json.loads(response.data.decode("utf-8"))) + response_as_json = json.loads(response.data.decode("utf-8")) + fetcher_name = response_as_json["fetchers"][0]["name"] + self.assertEqual("kiskadee-fetcher", fetcher_name) if __name__ == '__main__': From a275c57f5d0fa2069d43057881a1d856a6b4e463 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 24/27] Try to uncompress the source file. --- diff --git a/kiskadee/runner.py b/kiskadee/runner.py index e115353..50757e4 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -131,11 +131,8 @@ class Runner: ) ) uncompressed_source_path = tempfile.mkdtemp() - shutil.unpack_archive(compressed_source, uncompressed_source_path) - kiskadee.logger.debug( - 'ANALYSIS: Unpacking {} source in {} path' - .format(package['name'], uncompressed_source_path) - ) + try: + shutil.unpack_archive(compressed_source, uncompressed_source_path) kiskadee.logger.debug( 'ANALYSIS: Unpacking {} source in {} path' .format(package['name'], uncompressed_source_path) From b42d1e11d98f54192142e6be5978db1175730aed Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 25/27] Update README.md --- diff --git a/README.md b/README.md index deaba22..11e4f3d 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,14 @@ into a Firehose database. ## Setup -### Dependencies +To run kiskadee, you must have docker installed and running. Use the +dockerfiles in the `util` directory to build the images for each static +analyzer. The name of the image must be equal of the analyzer name. +You can accomplish that by doing -The name of the dependencies are compatible -with the Fedora distribution. If you use another operational system, -you will have to find the compatible names for the dependencies. -The `redhat-rpm-config` -package, is a specific Fedora dependency, if you not use Fedora (or a -Red Hat distribution), maybe you will not have to install it. + docker build . -t cppcheck + +With the Docker images build, create a virtualenv to kiskadee `dnf` is a package manager for the Fedora distribution (On Debian and Ubuntu is apt), @@ -25,7 +25,11 @@ to install the dependencies below. - redhat-rpm-config python-pip - python-pip -### Virtual Environment +Install some package dependencies. The name of the dependencies are compatible +with the Fedora distribution. If you use another distribution, you will have +to find the compatible name for the dependencies. The `redhat-rpm-config` +package, is a specific Fedora dependency. If you are not in Fedora (or a +Red Hat distribution), maybe you will not have to install it. Create a [virtualenv](https://virtualenv.pypa.io/en/stable/) to kiskadee. The virtualenv package will create a isolated environment @@ -35,94 +39,14 @@ for our python dependencies. virtualenv -p /usr/bin/python3 . source bin/activate -Install the python dependencies using pip +Kiskadee use postgresql as database. You will need to create a database named +kiskadee, with a role kiskadee as owner. + +Install python dependencies and run kiskadee pip install -e . pip install "fedmsg[consumers]" -### Docker Images - -To run the static analyzers, you must have -[Docker](https://www.docker.com/community-edition) installed and running. -If you have configured the Docker engineer properly, -run the *docker_build.sh* script. It will build the images for you. - - chmod u+x docker_build.sh - ./docker_build.sh - -### Database -Now we will create the kiskadee database. You will need to install the -postgresql packages for your system. If you use Fedora, follow the next -steps, if not, you will have to find out how install postgresql on your -system. - - sudo dnf install postgresql-server postgresql-contrib - sudo systemctl enable postgresql - sudo postgresql-setup initdb - sudo systemctl start postgresql - -To install on Ubuntu use this [link](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04). - -With postgresql installed, you will need to create the kiskadee role and -database. - - sudo su - postgres - createdb kiskadee - createuser kiskadee -P - # use kiskadee as password. - psql -U postgres -c "grant all privileges on database kiskadee to kiskadee" - # go back to your user (ctrl+d) - echo "localhost:5432:kiskadee:kiskadee:kiskadee" > ~/.pgpass - chmod 600 ~/.pgpass - -Restart the postgresql service: - - sudo systemctl restart postgresql - -Test the database connection: - - psql -U kiskadee -d kiskadee - -If you was not able to log in on the database, you will need to edit -the *pg_hba.conf* and change some rules defined by the postgresql package. -On Linux systems this file normally stays at the -`/var/lib/pgsql/data/`. Open this file and change: - - # "local" is for Unix domain socket connections only - local all all peer - # IPv4 local connections: - host all all 127.0.0.1/32 ident - # IPv6 local connections: - host all all ::1/128 ident - -to: - - # "local" is for Unix domain socket connections only - local all all md5 - # IPv4 local connections: - host all all 127.0.0.1/32 md5 - # IPv6 local connections: - host all all ::1/128 md5 - - -After this change, restarts the postgresql service: - - sudo systemctl restart postgresql - -Test the database connection: - - psql -U kiskadee -d kiskadee - -If you was able to get into the psql shell, the database is properly -configured. Leave the shell with ctrl+d. - -### Running our first analysis - -Kiskadee reads environment variables from the `util/kiskadee.conf` file. -If everything goes well till now, open the *kiskadee.conf* file, and set as -active (`active = yes`) only the *example_fetcher*, the other fetchers will -stay as `active = no`. - Now run kiskadee by typing `kiskadee` on the terminal. If the Docker images was properly build, and the Docker client was properly configured on your machine, kiskadee will be able to analysis a @@ -132,33 +56,50 @@ Kiskadee will decompress the example source, and run the analyzers defined on the *kiskadee.conf* file. You can use any postgresql client to access the database that you have created, and check the analysis maded by kiskadee. -### Running API +Kiskadee looks for its configuration file under `util/kiskadee.conf`. +If everything goes well till now, open the kiskadee.conf file, and set as +active only the example fetcher. Now run kiskadee by typing `kiskadee` on +the terminal. If the Docker images was properly build, and the Docker client +was properly configured on your machine, kiskadee will be able to analysis a +exemple source code. This code is in the kiskadee/tests/test\_source/ directory. -To run the kiskadee api just execute the command: +To run the API just run the command `kiskadee_api`. - kiskadee_api +### Anitya Fetcher +If you intend to run the anitya fetcher, you will have to install fedmsg-hub, +in order to kiskadee be able to consume the fedmsg events. +To install fedmsg-hub follow this steps inside the kiskadee root path: -## Tests and coverage + # Run this inside the kiskadee's virtualenv + sudo mkdir -p /etc/fedmsg.d/ + sudo cp util/base.py util/endpoints.py /etc/fedmsg.d/ + sudo cp util/anityaconsumer.py /etc/fedmsg.d/ + PYTHONPATH=`pwd` fedmsg-hub -To check kiskadee tests and coverage just run: +With this steps, fedmsg-hub will instantiate `AnityaConsumer` and publish +the monitored events using ZeroMQ. When kiskadee starts it will consume +the messages published by the consumer, and will run the analysis. - chmod u+x run_tests_and_coverage.sh - ./run_tests_and_coverage.sh +The events that comes to the anitya fetcher are published by Anitya, on this +[page](https://apps.fedoraproject.org/datagrepper/raw?category=anitya.) -To check kiskadee coverage open the file *covhtml/index.html*. +For more info about the Anitya service, read kiskadee documentation. -## Repositories +### Debian Fetcher +If you intend to use the debian fetcher, you will have to install the +`devscripts` package, in order use the necessary Debian tools to run the +fetcher. + +## Development Kiskadee daemon and API development are hosted at [pagure](https://pagure.io/kiskadee). Kiskadee frontend is hosted at [pagure](https://pagure.io/kiskadee/kiskadee_ui). Feel free to open issues and pull requests there. -We also have mirrors on [gitlab](https://gitlab.com/kiskadee/kiskadee) and +We also have mirrors on [gitlab](https://gitlab.com/kiskadee/kiskadee) and [github](https://github.com/LSS-USP/kiskadee). -Kiskadee have a CI environment hosted at this [url](http://143.107.45.126:30130/blue/organizations/jenkins/LSS-USP%2Fkiskadee/activity). - ## Documentation [kiskadee documentation is hosted at pagure.](docs.pagure.org/kiskadee) @@ -167,36 +108,9 @@ To build the documentation just entry in the doc directory, and run make html -To access the documentation open the `index.html` file, inside the +To access the documentation open the `index.html` file, inside the doc/\_build/html. -## Fetchers - -### Debian Fetcher -If you intend to use the debian fetcher, you will have to install the -`devscripts` package, in order use the necessary Debian tools to run the -fetcher. - -### Anitya Fetcher -If you intend to run the anitya fetcher, you will have to install fedmsg-hub, -in order to kiskadee be able to consume the fedmsg events. -To install fedmsg-hub follow this steps inside the kiskadee root path: - - # Run this inside the kiskadee's virtualenv - sudo mkdir -p /etc/fedmsg.d/ - sudo cp util/base.py util/endpoints.py /etc/fedmsg.d/ - sudo cp util/anityaconsumer.py /etc/fedmsg.d/ - PYTHONPATH=`pwd` fedmsg-hub - -With this steps, fedmsg-hub will instantiate `AnityaConsumer` and publish -the monitored events using ZeroMQ. When kiskadee starts it will consume -the messages published by the consumer, and will run the analysis. - -The events that comes to the anitya fetcher are published by Anitya, on this -[page](https://apps.fedoraproject.org/datagrepper/raw?category=anitya.) -For more info about the Anitya service, read kiskadee documentation. - - ## License Copyright (C) 2017 the AUTHORS (see the AUTHORS file) diff --git a/doc/architecture.rst b/doc/architecture.rst index 8e22ca5..1dbd1c0 100644 --- a/doc/architecture.rst +++ b/doc/architecture.rst @@ -27,4 +27,3 @@ and this was a scope decision made by the kiskadee community. .. *Figure One: Kiskadee architecture.* - diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 50757e4..f3b6b80 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -132,7 +132,10 @@ class Runner: ) uncompressed_source_path = tempfile.mkdtemp() try: - shutil.unpack_archive(compressed_source, uncompressed_source_path) + shutil.unpack_archive( + compressed_source, + uncompressed_source_path + ) kiskadee.logger.debug( 'ANALYSIS: Unpacking {} source in {} path' .format(package['name'], uncompressed_source_path) From 3aafc7a2ffecd5d51abe4d6f7b9a791f47b09666 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 24 2017 21:15:59 +0000 Subject: [PATCH 26/27] Add CI link in README.me --- diff --git a/README.md b/README.md index 11e4f3d..d4c2c23 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Feel free to open issues and pull requests there. We also have mirrors on [gitlab](https://gitlab.com/kiskadee/kiskadee) and [github](https://github.com/LSS-USP/kiskadee). +Kiskadee have a CI environment hosted at this [url](http://143.107.45.126:30130/blue/organizations/jenkins/LSS-USP%2Fkiskadee/activity). ## Documentation [kiskadee documentation is hosted at pagure.](docs.pagure.org/kiskadee) From f1700cd38b5ae5712d5c91b10e9151fc9466ea16 Mon Sep 17 00:00:00 2001 From: gabrielsclimaco Date: Aug 24 2017 21:16:00 +0000 Subject: [PATCH 27/27] Add docker for environment set up and documentation for running it --- diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9414382 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9cb82e8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM fedora + +RUN curl -o docker.rpm https://download.docker.com/linux/fedora/24/x86_64/stable/Packages/docker-ce-17.06.0.ce-1.fc24.x86_64.rpm &&\ + dnf install -y openssl-devel python3-devel gcc redhat-rpm-config python-pip docker.rpm &&\ + mkdir /app + +ADD . /app +WORKDIR /app + +RUN pip install virtualenv && virtualenv -p /usr/bin/python3 . &&\ + source bin/activate && pip install -e . && pip install "fedmsg[consumers]" + +RUN source bin/activate diff --git a/README.md b/README.md index d4c2c23..fc900d7 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,125 @@ Install python dependencies and run kiskadee pip install -e . pip install "fedmsg[consumers]" +### Docker Images + +To run the static analyzers, you must have +[Docker](https://www.docker.com/community-edition) installed and running. +If you have configured the Docker engineer properly, +run the *docker_build.sh* script. It will build the images for you. + + chmod u+x docker_build.sh + ./docker_build.sh + +### Database +Now we will create the kiskadee database. You will need to install the +postgresql packages for your system. If you use Fedora, follow the next +steps, if not, you will have to find out how install postgresql on your +system. + + sudo dnf install postgresql-server postgresql-contrib + sudo systemctl enable postgresql + sudo postgresql-setup initdb + sudo systemctl start postgresql + +To install on Ubuntu use this [link](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04). + +With postgresql installed, you will need to create the kiskadee role and +database. + + sudo su - postgres + createdb kiskadee + createuser kiskadee -P + # use kiskadee as password. + psql -U postgres -c "grant all privileges on database kiskadee to kiskadee" + # go back to your user (ctrl+d) + echo "localhost:5432:kiskadee:kiskadee:kiskadee" > ~/.pgpass + chmod 600 ~/.pgpass + +Restart the postgresql service: + + sudo systemctl restart postgresql + +Test the database connection: + + psql -U kiskadee -d kiskadee + +If you was not able to log in on the database, you will need to edit +the *pg_hba.conf* and change some rules defined by the postgresql package. +On Linux systems this file normally stays at the +`/var/lib/pgsql/data/`. Open this file and change: + + # "local" is for Unix domain socket connections only + local all all peer + # IPv4 local connections: + host all all 127.0.0.1/32 ident + # IPv6 local connections: + host all all ::1/128 ident + +to: + + # "local" is for Unix domain socket connections only + local all all md5 + # IPv4 local connections: + host all all 127.0.0.1/32 md5 + # IPv6 local connections: + host all all ::1/128 md5 + + +After this change, restarts the postgresql service: + + sudo systemctl restart postgresql + +Test the database connection: + + psql -U kiskadee -d kiskadee + +If you was able to get into the psql shell, the database is properly +configured. Leave the shell with ctrl+d. + +### With Docker + +If you don't want to install all dependencies, use the Dockerfile on the root of the project: + +1. First, build the image: + +``` +docker build -t kiskadee_backend . +``` + +2. Then, change the execution permissions of the docker shell script: + +``` +chmod +x run_docker.sh +``` + +* 3. Run the shell script + +``` +./run_docker.sh +``` + +* 4. Now you're into docker, just enter the environment as usual: + +``` +source bin/activate +``` + +Now you can run ```kiskadee```. + +--- + +**Obs:** You still need to set up the [PostgreSQL database](#database) and build the [docker images](#docker-images). + +--- + +### Running our first analysis + +Kiskadee reads environment variables from the `util/kiskadee.conf` file. +If everything goes well till now, open the *kiskadee.conf* file, and set as +active (`active = yes`) only the *example_fetcher*, the other fetchers will +stay as `active = no`. + Now run kiskadee by typing `kiskadee` on the terminal. If the Docker images was properly build, and the Docker client was properly configured on your machine, kiskadee will be able to analysis a diff --git a/run_docker.sh b/run_docker.sh new file mode 100755 index 0000000..cd82d24 --- /dev/null +++ b/run_docker.sh @@ -0,0 +1,13 @@ +docker run --rm -it \ + -v "/var/run/docker.sock:/var/run/docker.sock" \ + -v "$(pwd)/kiskadee:/app/kiskadee" \ + -v "$(pwd)/util:/app/util" \ + --net="host" \ + kiskadee_backend bash + +# line 1 - Run container iteratively without saving its instance +# line 2 - Map docker.sock to make docker in docker possible +# line 3 - Map kiskadee folder +# line 4 - Map util folder +# line 5 - Make sure the container run in the same host as the docker to connect +# with PostgreSQL via port 5432