code polish, Makefile and README update

Author: Janez Justin

rpm/Makefile

@@ -1,10 +1,19 @@
-ZIPPED			:= s3rpm.py gnupg.py pyrpm/* pyrpm/tools/*
+ZIPPED_FILES := s3rpm.py gnupg.py # files to compress in root of zip
+ZIPPED_DIRS  := pyrpm # folders to compress to zip
+all: requires test package
 
+help: ## displays this message
+	@grep -E '^[a-zA-Z_/%\-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-set: requires package
+requires: ## installs required packages
+	pip3 install -t . -r requirements.txt
 
-requires:
-	pip3 install -t . -r requirements.txt --upgrade
-	
-package:
-	zip code.zip $(ZIPPED)
+package: ## creates zip of code
+	zip code.zip $(ZIPPED_FILES)
+	zip -r code.zip $(ZIPPED_DIRS)
+
+clean: ## cleans up the repository
+	/bin/rm -rf code.zip
+
+test: ## runs the tests
+	python3.6 s3rpm_test.py

rpm/README.md

@@ -2,36 +2,68 @@
 
 Automatic YUM repository building inside S3 bucket using with lambda support
 
-## Setting up S3 and Lambda
-
-Clone the repo and get all other required files
+## Readme contents
+
+* [Setting up code, S3 and Lambda](#setting-up-code-s3-and-lambda)
+    * [Getting the code](#getting-the-code)
+    * [GPG key](#gpg-key)
+    * [Environmental variables](#environmental-variables)
+    * [Set up role](#set-up-role)
+    * [Set up lambda with CLI](#set-up-lambda-with-cli)
+    * [Set up lambda manually](#set-up-lambda-manually)
+    * [The triggers](#the-triggers)
+    * [Set up S3](#set-up-s3)
+* [Setting up yum](#setting-up-yum)
+    * [First time set up](#first-time-set-up)
+    * [Install/update](#installupdate)
+* [Notes](#notes)
+* [Tests](#tests)
+
+## Setting up code, S3 and Lambda
+
+### Getting the code
+Clone the repo, get all other required files and compress them
 ```
 git clone https://github.com/tactycal/lambdaRepos.git
 cd lambdaRepos/rpm
-pip3 install -t . -r requirements.txt
+make all
 ```
 
-Compress all needed files
+### GPG key
+create your gpg key (skip to exporting your key, if you already have it)
 ```
-zip code.zip s3rpm.py gnupg.py pyrpm/* pyrpm/tools/*
+gpg --gen-key
+# Follow the instructions
+# Create 'RSA and RSA' key - option 1
+# For maxium encryption it is recommended to make 4096 bits long key
+# Key should not expire
 ```
 
-Or just use `make set` instead of `zip` and `pip3` command
+export your key
 
-Presuming you already have GPG key generated export secret key (you can skip this part if you don't want to GPG sign your repository)
 ```
-gpg -a --export-secret-key > secret.key
+gpg --export-secret-key -a "User Name" > secret.key  # exports secret key to secret.key
 ```
 
-Create new lambda function, set handler to **s3rpm.lambda_handler**, runtime to **python 3.6** and the triggers to:
+### Set up role
 
- * Object Created, suffix 'rpm'
- * Object Removed, suffix 'rpm'
- * If you are using certain directory as a repo, set it as prefix
+Create new role with s3 write/read access
 
-Upload `code.zip` to lambda function
+Here is a minimal requirement for the policy that is included in role:
+```
+{"Version": "2012-10-17",
+    "Statement": [
+        {"Sid": "<THIS IS UNIQE>",
+            "Action": [
+                "s3:GetObject",
+                "s3:PutObject",
+                "s3:PutObjectAcl"],
+            "Effect": "Allow",
+            "Resource": "arn:aws:s3:::<YOUR BUCKET NAME>/*"}]}
+```
 
-Set the environmental variables
+### Environmental variables
+These are the environmental variables you will have to set:
 
 | Key | Value |
 | --- | ---|
@@ -51,14 +83,44 @@ Set the environmental variables
 
 **REPO_DIR** Path to repositroy from bucket root. If none is set, it is assumed root of repository is root of the bucket
 
+### Set up lambda with CLI
+
+[Install aws cli](http://docs.aws.amazon.com/cli/latest/userguide/installing.html)
+
+Create new lambda function:
+```
+aws lambda create-function \
+    --function-name <name the function> \
+    --zip-file fileb://code.zip \
+    --role <role's arn> \    # arn from role with S3 read/write access
+    --handler s3rpm.handler \
+    --runtime python3.6 \
+# Replace '<...>' with environmental variables
+    --environment Variables='{PUBLIC=<bool>, GPG_KEY=<file>, GPG_PASS=<password>, BUCKET_NAME=<bucket name>, REPO_DIR=<dir>}'
+```
+
+### Set up lambda manually
 
+If CLI is not your thing, then you can upload code manaully
+
+Create new lambda function, set handler to **s3rpm.lambda_handler**, runtime to **python 3.6**
+
+Upload `code.zip` to lambda function
+
+### The triggers
+
+ * Object Created(All), suffix 'rpm'
+ * Object Removed(All), suffix 'rpm'
+ * If you are using certain directory as a repo, set it as prefix
+
+### Set up S3
 Upload secret key file to location you specified as GPG_KEY
 
-Upload GPG SIGNED .rpm file to desired folder, lambda function should now keep your repository up to date
+Upload .rpm file to desired folder, lambda function should now keep your repository up to date
 
 ## Setting up yum
 
-**First time set up**
+### First time set up
 
 create `example.repo` file in `/etc/yum.repos.d/example.repo` 
 ```
@@ -78,23 +140,29 @@ gpgkey=<link to public key of key you used for signing metadata files>
 * You can do `repo_gpgcheck=0` to skip gpg verification when installing packages
 * You can do `gpgcheck=1` if you are uploading signed rpm packages(lambda does not sign them, it signs only metadata xml file)
 
+### Install/update
 Install package
 ```
-su
-yum install <package name>
+sudo yum install <package name>
 ```
 
 Upgrading package
 ```
-su
-yum upgrade
+sudo yum upgrade
 ```
 
 ## Notes
 
-.rpm and repodata/* in repository directory are and should be publicly accessible
+* .rpm and repodata/* in repository directory are and should be publicly accessible for the 
+
+* Don't forget to increase the timeout of lambda function
 
-Don't forget to increase the timeout of lambda function
+* If somebody tries to inject a malicious rpm file in your repo it will be automaticly added to repository. It is your job to make bucket secure enough for this not to happen.!!!
 
-If somebody tries to inject a malicious rpm file in your repo it will be automaticly added to repository. It is your job to make bucket secure enough for this not to happen.!!!
+## Tests
 
+To run unit tests:
+```
+make requires   #gets dependancies
+make test       #runs the tests
+```

rpm/s3rpm.py

@@ -15,16 +15,13 @@ def lambda_handler(event, context):
     repo = YumRepository('/tmp/repo/') # set repository
     prefix = '/'.join(key.split('/')[0:-1])+'/'
 
-    if os.environ['REPO_DIR'].endswith('/'):
-        os.environ['REPO_DIR'] = os.environ['REPO_DIR'][:-1]
-    if os.environ['REPO_DIR'].startswith('/'):
-        os.environ['REPO_DIR'] = os.environ['REPO_DIR'][1:]
+    s3_repo_dir = os.environ['REPO_DIR'].strip('/')
 
     #make sure we are working with correct files
-    if bucket == os.environ['BUCKET_NAME'] and key.endswith(".rpm") and prefix.startswith(os.environ['REPO_DIR']):
+    if bucket == os.environ['BUCKET_NAME'] and key.endswith(".rpm") and prefix.startswith(s3_repo_dir):
         #check if repodata already exist, if not create new with key file
         print('Bucket and key\'s file accepted')
-        exists = check_bucket_file_existance(os.environ['REPO_DIR']+'/repodata/repomd.xml')
+        exists = check_bucket_file_existance(s3_repo_dir+'/repodata/repomd.xml')
         files = ['repomd.xml', 'primary.xml.gz','filelists.xml.gz', 'other.xml.gz']
 
         #make /tmp/repodata path
@@ -33,26 +30,26 @@ def lambda_handler(event, context):
         if exists: 
             print('repodata already exists, old files will be overwriten')
             for f in files:
-                s3.download_file(os.environ['BUCKET_NAME'], os.environ['REPO_DIR']+'/repodata/'+f, repo.repodir+'repodata/'+f)
+                s3.download_file(os.environ['BUCKET_NAME'], s3_repo_dir+'/repodata/'+f, repo.repodir+'repodata/'+f)
             repo.read()
         print('Creating Metadata files')
-        repo, cache = check_changed_files(repo)
+        repo, cache = check_changed_files(repo, s3_repo_dir)
         #Check if object was removed
 
         repo.save()
 
         #sign metadata
         if not os.environ['GPG_KEY']=='':
-            sign_md_file(repo)
+            sign_md_file(repo, s3_repo_dir)
 
         #save files to bucket
         s3 = boto3.resource('s3')
         for f in files:
             with open(repo.repodir+'repodata/'+f, 'rb') as g:
-                f_index_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=os.environ['REPO_DIR']+'/repodata/'+f)
+                f_index_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=s3_repo_dir+'/repodata/'+f)
                 print("Writing file: %s" % (str(f_index_obj)))
                 f_index_obj.put(Body=g.read(-1), ACL=get_public())
-        f_index_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=os.environ['REPO_DIR']+'/repo_cache')
+        f_index_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=s3_repo_dir+'/repo_cache')
         print("Writing file: %s" % (str(f_index_obj)))
         f_index_obj.put(Body=str(json.dumps(cache)))
 
@@ -102,40 +99,40 @@ def get_public():
         acl = 'private'
     return acl
 
-def get_cache(repo):
+def get_cache(repo, s3_repo_dir):
     """
     Check for cache file
     """
-    if check_bucket_file_existance(os.environ['REPO_DIR']+'/repo_cache'):
-        print('Repodata cache (%s) found, attempting to write to it' %(os.environ['REPO_DIR']+'/repo_cache'))
+    if check_bucket_file_existance(s3_repo_dir+'/repo_cache'):
+        print('Repodata cache (%s) found, attempting to write to it' %(s3_repo_dir+'/repo_cache'))
         s3 = boto3.client('s3')
-        s3.download_file(os.environ['BUCKET_NAME'], os.environ['REPO_DIR']+'/repo_cache', repo.repodir + 'repo_cache')
+        s3.download_file(os.environ['BUCKET_NAME'], s3_repo_dir+'/repo_cache', repo.repodir + 'repo_cache')
         with open(repo.repodir + 'repo_cache', 'r') as f:
             cache = json.loads(f.read(-1))
     else:
         print('repodata_cache file doesn\'t exist. Creating new one')
         cache = {}
     return cache
 
-def check_changed_files(repo):
+def check_changed_files(repo, s3_repo_dir):
     """
     check if there are any new files in bucket or any deleted files
     """
-    print("Checking for changes : %s" % (os.environ['REPO_DIR']))
-    cache = get_cache(repo)
+    print("Checking for changes : %s" % (s3_repo_dir))
+    cache = get_cache(repo, s3_repo_dir)
     s3 = boto3.resource('s3')
     files = []
     #cycle through all objects ending with .rpm in REPO_DIR and check if they are already in repodata, if not add them
-    for obj in s3.Bucket(os.environ['BUCKET_NAME']).objects.filter(Prefix=os.environ['REPO_DIR']):
+    for obj in s3.Bucket(os.environ['BUCKET_NAME']).objects.filter(Prefix=s3_repo_dir):
         files.append(obj.key)
         if not obj.key.endswith(".rpm"):
             print('skipping %s - not rpm file' %(obj.key))
             continue
-        fname = obj.key[len(os.environ['REPO_DIR']):] # '/filename.rpm' - without path
+        fname = obj.key[len(s3_repo_dir):] # '/filename.rpm' - without path
         if fname not in cache:
             s3c = boto3.client('s3')
             #Create path to folder where to download file, if it not yet exists
-            prefix = '/'.join(obj.key.split('/')[0:-1])[len(os.environ['REPO_DIR']):]
+            prefix = '/'.join(obj.key.split('/')[0:-1])[len(s3_repo_dir):]
             create_new_dir_if_not_exist(repo.repodir+prefix)
             #Download file to repodir
             path = repo.repodir + fname
@@ -151,7 +148,7 @@ def check_changed_files(repo):
 
     removedPkgs = []
     for f in cache:
-        if f.endswith('.rpm') and os.environ['REPO_DIR']+f not in files:
+        if f.endswith('.rpm') and s3_repo_dir+f not in files:
             print('removing ' +f)
             repo = remove_pkg(repo, cache, f)
             removedPkgs.append(f)
@@ -173,7 +170,7 @@ def remove_pkg(repo, cache, key):
         print('Tried to delete %s entry but was not found in cache' % (filename))
     return repo
 
-def sign_md_file(repo):
+def sign_md_file(repo, s3_repo_dir):
     '''
     Using gpg password assigned in env variable `GPG_PASS` and key, which's file directory is 
     assigned in env variable `GPG_KEY`
@@ -190,6 +187,6 @@ def sign_md_file(repo):
         signed = gpg.sign_file(stream, passphrase=os.environ['GPG_PASS'], clearsign=True, detach=True, binary=False)
 
     s3 = boto3.resource('s3')
-    sign_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=os.environ['REPO_DIR'] + "/repodata/repomd.xml.asc")
+    sign_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=s3_repo_dir + "/repodata/repomd.xml.asc")
     print('uploading repomd.xml.asc to /repodata')
     sign_obj.put(Body=str(signed), ACL=get_public())

rpm/s3rpm_test.py

@@ -66,13 +66,13 @@ def test_cache(self, check_mock, s3_mock, yum_mock):
         check_mock.return_value = True
 
         with patch('s3rpm.open', m):
-            cachenew = s3rpm.get_cache(repo)
+            cachenew = s3rpm.get_cache(repo, os.environ['REPO_DIR'])
         s3_mock.client().download_file.assert_called_with('bucket', 'test_s3rpm/repo_cache', 'test_s3rpm/repo_cache')
         self.assertEqual(json.loads(cache), cachenew)
 
         check_mock.return_value = False
 
-        cachenew = s3rpm.get_cache(repo)
+        cachenew = s3rpm.get_cache(repo,os.environ['REPO_DIR'])
         self.assertEqual(cachenew, {})
 
     @patch('s3rpm.YumRepository')
@@ -91,7 +91,7 @@ def test_new_files(self, s3_mock, cache_mock, yump_mock, yum_mock):
         s3_mock.resource().Bucket().objects.filter.return_value = [MagicMock(key='test.file'),MagicMock(key='test_s3rpm/pkgname-0.3.8-x86_64.rpm'), MagicMock(key='test_s3rpm/pkgname-0.3.7-x86_64.rpm')]
         m = mock_open(read_data='')
         with patch('s3rpm.open', m):
-            reponew, cachenew = s3rpm.check_changed_files(repo)
+            reponew, cachenew = s3rpm.check_changed_files(repo, os.environ['REPO_DIR'])
 
         self.assertEqual(cache, cachenew)
         self.assertEqual(yum_mock.add_package.call_count, 1)
@@ -107,7 +107,7 @@ def test_delete_files(self, s3_mock, cache_mock, yum_mock):
         cache = {}
 
         s3_mock.resource().Bucket().objects.filter.return_value = [MagicMock(key='test.file')]
-        _, cachenew = s3rpm.check_changed_files(repo)
+        _, cachenew = s3rpm.check_changed_files(repo, os.environ['REPO_DIR'])
         self.assertEqual(cache, cachenew)
         self.assertEqual(yum_mock.remove_package.call_count, 1)
 
@@ -119,7 +119,7 @@ def test_gpg(self, s3_mock, gpg_mock, yum_mock):
         m = mock_open()
         repo = yum_mock()
         with patch('s3rpm.open', m):
-            s3rpm.sign_md_file(repo)
+            s3rpm.sign_md_file(repo, os.environ['REPO_DIR'])
             gpg_mock.GPG().sign_file.assert_called_with(s3rpm.open(), binary=False, clearsign=True, detach=True, passphrase='123')
         s3_mock.resource().Object().put.assert_called_with(ACL='public-read', Body=str(gpg_mock.GPG().sign_file()))
 
@@ -159,7 +159,7 @@ def test_defined_repodir(self, s3_mock, yum_mock, cache_mock, ):
     @patch('s3rpm.get_cache')
     @patch('s3rpm.YumRepository')
     @patch('s3rpm.boto3')
-    def test_gpg_test(self, s3_mock, yum_mock, cache_mock, sh_mock, gpg_mock):
+    def test_gpg_from_handler(self, s3_mock, yum_mock, cache_mock, sh_mock, gpg_mock):
         cache_mock.return_value = {"pkgname":"ID"}
 
         os.environ['GPG_KEY'] = 'KeyNowExists'
@@ -173,11 +173,9 @@ def test_gpg_test(self, s3_mock, yum_mock, cache_mock, sh_mock, gpg_mock):
         assert os.path.exists('test_s3rpm/testrepo/') == True
 
     @patch('s3rpm.boto3')
-    def test_bad_repo_dir_and_bucket_name(self, s3_mock):
-        os.environ['REPO_DIR'] = '/test/repo/'
+    def test_bad_bucket_name(self, s3_mock):
         os.environ['BUCKET_NAME'] = 'iamfakebucket'
         s3rpm.lambda_handler(S3_EVENT, {})
-        self.assertEqual(os.environ['REPO_DIR'], 'test/repo')
         s3_mock.client.assert_called_with('s3')
         self.assertEqual(len(s3_mock.resource().Object().put.mock_calls), 0)
 S3_EVENT = {"Records":[{"s3": {"object": {"key": "test_s3rpm/repo/pkgname-0.3.8-x86_64.rpm",},"bucket": {"name": "bucket",},},"eventName": "ObjectCreated:Put"}]}