Update tests and fix SizeFSAlphaNum generator bug when reading >64*1024 bytes

Author: Joel Wright

README.md

@@ -42,91 +42,86 @@ Create Size File objects in memory:
 
     from sizefs import SizeFS
     sfs = SizeFS()
-    sfs.read('/1B', 1, 0, None)
-    sfs.read('/2B', 2, 0, None)
-    sfs.read('/2K', 1024, 0, None)
-    sfs.read('/128K', 1024*128, 0, None)
-    sfs.read('/4G', 4*1024*1024, 0, None)
+    sfs.read('/ones/1B', 1, 0, None, create=True)
+    sfs.read('/ones/2B', 2, 0, None, create=True)
+    sfs.read('/ones/2K', 1024, 0, None, create=True)
+    sfs.read('/ones/128K', 1024*128, 0, None, create=True)
+    sfs.read('/ones/4G', 4*1024*1024, 0, None, create=True)
 
 The folder structure is used to determine the content of the files:
 
-    sfs.read('/zeros/5B', 5, 0, None).read(0, 5)
+    sfs.read('/zeros/5B', 5, 0, None, create=True).read(0, 5)
     out> 00000
 
-    sfs.read('/ones/5B', 5, 0, None).read(0, 5)
+    sfs.read('/ones/5B', 5, 0, None, create=True).read(0, 5)
     out> 11111
 
-    sfs.read('/alpha_num/5B', 5, 0, None).read(0, 5)
+    sfs.read('/alpha_num/5B', 5, 0, None, create=True).read(0, 5)
     out> TMdEv
 
-Folders can be created to manipulate the data:
+The folders 'ones', 'zeros' and 'alpha_num' are always present,
+but new folders can also be created. When files are created in a
+folder, the xattrs of the folder determine that file's content until
+the file's xattrs are updated:
 
     sfs.mkdir('/regex1', None)
-    sfs.setxattr('/regex1', 'filler', '0', None)
-    print sfs.read('/alpha_num/5B', 5, 0, None).read(0, 5)
+    sfs.setxattr('/regex1', 'generator', 'regex', None)
+    sfs.setxattr('/regex1', 'filler', 'regex', None)
+    print sfs.read('/regex1/5B', 5, 0, None, create=True).read(0, 5)
 
-    out> 00000
+    out> regex
 
-    sfs.mkdir('/regex2', None)
-    sfs.setxattr('/regex2', 'filler', '1', None)
-    print sfs.read('/regex2/5B', 5, 0, None).read(0, 5)
+    sfs.setxattr('/regex1/5B', 'filler', 'string', None)
+    print sfs.read('/regex1/5B', 5, 0, None).read(0, 5)
 
-    out> 11111
+    out> string
 
-    sfs.mkdir('/regex3', None)
-    sfs.setxattr('/regex3', 'filler', '[a-zA-Z0-9]', None)
-    print sfs.read('/regex3/5B', 5, 0, None).read(0, 5)
+    sfs.setxattr('/regex1/5B', 'filler', 'a{2}b{2}c', None)
+    print sfs.read('/regex1/5B', 5, 0, None).read(0, 5)
 
-    out> 1JAbd
+    out> aabbc
 
-Files can be added to SizeFS using sfs.create:
+Files can also be added to SizeFS using sfs.create:
 
-    sfs.mkdir('/regex3', None)
-    sfs.setxattr('/regex3', 'filler', '[a-zA-Z0-9]', None)
-    sfs.create('/regex3/5B', None)
-    print sfs.read('/regex3/5B', 5, 0, None)
+    sfs.mkdir('/folder', None)
+    sfs.create('/folder/5B', None)
+    print sfs.read('/folder/5B', 5, 0, None)
+
+    out> 11111
 
-    out> aS8yG
+And as discussed above, the name of the file determines its size:
 
     sfs.create('/regex3/128K', None)
-    print len(sfs.read('/regex3/128K', 128*1024, 0, None))
+    # Try to read more contents than the files contains
+    print len(sfs.read('/regex3/128K', 256*1000, 0, None))
 
-    out> 131072
+    out> 128000
 
     sfs.create('/regex3/128K-1B', None)
-    print len(sfs.read('/regex3/128K-1B', 128*1024, 0, None))
+    # Try to read more contents than the files contains
+    print len(sfs.read('/regex3/128K-1B', 256*1000, 0, None))
 
-    out> 131071
+    out> 127999
 
     sfs.create('/regex3/128K+1B', None)
-    print len(sfs.read('/alphanum/128K+1B', 128*1024+1, 0, None))
-
-    out> 131073
-
-File content can be generated that matches a regex pattern by adding a directory
-
-    sfs.mkdir('/regex1')
-    sfs.setxattr('/regex1','filler','a(bcd)*e{4}[a-z03]*')
-    sfs.create('/regex1','128K')
-    print len(sfs.open('regex1/128KB').read(0, 128*1024))
+    # Try to read more contents than the files contains
+    print len(sfs.read('/alphanum/128K+1B', 256*1000, 0, None))
 
-    out> 131072
-
-    sfs.create('/regex1','128K-1B')
-    print len(sfs.open('regex1/128K-1B').read(0, 128*1024-1))
-
-    out> 131071
-
-    sfs.create('/regex1','128K+1B')
-    print len(sfs.open('regex1/128KB+1B').read(0, 128*1024+1))
-
-    out> 131073
+    out> 128001
 
 
 Extended Usage
 --------------
 
-We can set up to 5 properties:
+The 'generator' xattr property defines the file content and can be set to one
+of:
+
+    ones       - files are filled with ones
+    zeros      - files are filled with zeros
+    alpha_num  - files are filled with alpha numeric characters
+    regex      - files are filled according to a collection of regular expression patterns
+
+We can set up to 5 properties to control the regular expression patterns:
 
     prefix     - defined pattern for the start of a file (default = "")
     suffix     - defined pattern for the end of a file (default = "")

sizefs/__init__.py

@@ -1,3 +1,3 @@
-__all__ = ['SizeFS', 'SizeFile']
+__all__ = ['SizeFS']
 
-from sizefs import SizeFS, SIZEFSCHARS
+from sizefs import SizeFS

sizefs/contents.pyx

@@ -55,11 +55,20 @@ class SizeFSAlphaNumGen(object):
 
     def __init__(self):
         self.pre_seed = ''.join(random.choice(self.chars)
-                                for _x in range(64*1024))
+                                for _ in range(64*1024))
 
     def read(self, start, end):
         if start <= end:
-            return self.pre_seed[0:end-start+1]
+            toRead = end - start
+            pre_seed_count = 0
+            content = []
+            pre_seed_len = len(self.pre_seed)
+            while toRead > pre_seed_len:
+                content.append(self.pre_seed)
+                toRead -= pre_seed_len
+                pre_seed_count += 1
+            content.append(self.pre_seed[0:toRead+1])
+            return ''.join(content)
         else:
             return ''
 

sizefs/sizefs.py

@@ -169,7 +169,7 @@ def getattr(self, path, fh=None):
             if parent_folder in self.folders:
                 return self.folders[parent_folder]
 
-        raise FuseOSError(ENOENT) 
+        raise FuseOSError(ENOENT)
 
     def getxattr(self, path, name, position=0):
         """
@@ -229,15 +229,24 @@ def open(self, path, flags):
         self.fd += 1
         return self.fd
 
-    def read(self, path, size, offset, fh):
+    def read(self, path, size, offset, fh, create=False):
         """
         Returns content based on the pattern of the containing folder
         """
         if path in self.files:
-            content = self.files[path]['generator'].read(offset, offset+size-1)
-            return content
+            size_bytes = self.files[path]['attrs']['st_size']
+            if offset > (size_bytes - 1):
+                return ""
+            else:
+                end_of_content = min(offset+size-1, size_bytes-1)
+                content = self.files[path]['generator'].read(offset, end_of_content)
+                return content
         else:
-            raise FuseOSError(ENOENT) 
+            if not create:
+                raise FuseOSError(ENOENT)
+            else:
+                self.create(path, 0444)
+                return self.read(path, size, offset, fh, create=create)
 
     def readdir(self, path, fh):
         contents = ['.', '..']

tests/test_sfs.py

@@ -1,6 +1,7 @@
 __author__ = 'jjw'
 
-from sizefs import SizeFS, SIZEFSCHARS
+from sizefs import SizeFS 
+from sizefs.contents import SizeFSAlphaNumGen
 import unittest
 
 
@@ -10,30 +11,32 @@ def test_basic(self):
         # Basic Test
 
         sfs = SizeFS()
-        self.assertEqual(len(sfs.read('/zeros/1B', 1, 0, None)), 1)
-        self.assertEqual(len(sfs.read('/ones/1B', 1, 0, None)), 1)
-        self.assertEqual(len(sfs.read('/alpha_num/1B', 1, 0, None)), 1)
+        self.assertEqual(len(sfs.read('/zeros/1B', 1, 0, None, create=True)), 1)
+        self.assertEqual(len(sfs.read('/ones/1B', 1, 0, None, create=True)), 1)
+        self.assertEqual(len(sfs.read('/alpha_num/1B', 1, 0, None, create=True)), 1)
 
     def test_contents(self):
         # Contents Test
 
         sfs = SizeFS()
-        self.assertEqual(sfs.read('/zeros/5B', 5, 0, None), '00000')
-        self.assertEqual(sfs.read('/ones/5B', 5, 0, None), '11111')
-        for ch in sfs.read('/alpha_num/5B', 5, 0, None):
-            self.assertIn(ch, SIZEFSCHARS)
+        self.assertEqual(sfs.read('/zeros/5B', 5, 0, None, create=True), '00000')
+        self.assertEqual(sfs.read('/ones/5B', 5, 0, None, create=True), '11111')
+        for ch in sfs.read('/alpha_num/5B', 5, 0, None, create=True):
+            self.assertIn(ch, SizeFSAlphaNumGen.chars)
 
     def test_length(self):
         # Length Test
         k128 = 128*1000
+        k256 = 256*1000
         sfs = SizeFS()
-        self.assertEqual(len(sfs.read('/128B', 128, 0, None)), 128)
-        self.assertEqual(len(sfs.read('/128K', k128-1, 0, None)), k128-1)
-        self.assertEqual(len(sfs.read('/128K', k128, 0, None)), k128)
-        self.assertEqual(len(sfs.read('/128K+1B', k128+1, 0, None)), k128+1)
-        self.assertEqual(len(sfs.read('/zeros/5B', 5, 0, None)), 5)
-        self.assertEqual(len(sfs.read('/ones/5B', 5, 0, None)), 5)
-        self.assertEqual(len(sfs.read('/alpha_num/5B', 5, 0, None)), 5)
+        self.assertEqual(len(sfs.read('/zeros/128B', 128, 0, None, create=True)), 128)
+        self.assertEqual(len(sfs.read('/zeros/128K', k128-1, 0, None, create=True)), k128-1)
+        self.assertEqual(len(sfs.read('/alpha_num/128K', k128, 0, None, create=True)), k128)
+        self.assertEqual(len(sfs.read('/zeros/128K+1B', k128+1, 0, None, create=True)), k128+1)
+        self.assertEqual(len(sfs.read('/zeros/128K', k256, 0, None, create=True)), k128)
+        self.assertEqual(len(sfs.read('/zeros/5B', 5, 0, None, create=True)), 5)
+        self.assertEqual(len(sfs.read('/ones/5B', 5, 0, None, create=True)), 5)
+        self.assertEqual(len(sfs.read('/alpha_num/5B', 5, 0, None, create=True)), 5)
 
 
 if __name__ == '__main__':