Added fetch_size constructor param

CHANGELOG.rst

@@ -11,6 +11,10 @@ Unreleased
   ``dependson`` is now a list instead of a filter iterator. This fixes issue #72 
   where dependencies were only loaded in the first bulk load.
 
+**Changed**
+  ``SQLSource`` now has a ``fetch_size`` constructor parameter so the end-user can
+  control how much data should be held in main memory for each round trip to the RDBMS.
+
 Version 2.8
 -----------
 **Added**

pygrametl/datasources.py

@@ -106,7 +106,7 @@ class SQLSource(object):
     """A class for iterating the result set of a single SQL query."""
 
     def __init__(self, connection, query, names=(), initsql=None,
-                 cursorarg=None, parameters=None):
+                 cursorarg=None, parameters=None, fetch_size=500):
         """Arguments:
 
            - connection: the PEP 249 connection to use. NOT a
@@ -120,6 +120,8 @@ def __init__(self, connection, query, names=(), initsql=None,
              the connection's cursor method is called. Default: None.
            - parameters: if not None, this sequence or mapping of parameters
              will be sent when the query is executed.
+           - fetch_size: The amount of rows to fetch into memory for each round trip to the source.
+             All rows will be fetched at once if fetch_size is set to 0 or less.
         """
         self.connection = connection
         if cursorarg is not None:
@@ -145,7 +147,11 @@ def __iter__(self):
                     names = self.names or \
                         [t[0] for t in self.cursor.description]
             while True:
-                data = self.cursor.fetchmany(500)
+                if fetch_size <= 0:
+                    data = self.cursor.fetchall()
+                else:
+                    data = self.cursor.fetchmany(fetch_size)
+
                 if not data:
                     break
                 if not names:
@@ -160,6 +166,11 @@ def __iter__(self):
                         "%d given, %d needed." % (len(names), len(data[0])))
                 for row in data:
                     yield dict(zip(names, row))
+
+                # It is not well defined in PEP 249 what fetchall will do if called twice
+                # Therefore it is safest to break the loop if fetchall is used
+                if fetch_size <= 0:
+                    break
         finally:
             try:
                 self.cursor.close()