Add introduction ipynb

Author: reyoung

.gitignore

@@ -12,3 +12,4 @@ Makefile
 
 *~
 bazel-*
+.ipynb_checkpoints

demo/introduction/linear.ipynb

@@ -224,6 +224,7 @@ def run_one_pass(self):
                 break
             exit_flag = self.__end_batch__()
         self.__end_pass__()
+        return self.__context__
 
     def __enter__(self):
         """

paddle/py_paddle/trainer/base.py

@@ -16,17 +16,22 @@ def __init__(self, network, use_gpu=False, device_count=1):
         self.__train_data__ = None
         self.__updater__ = None
         self.__gradient_machine__ = None
+        self.__init_param__ = None
         self.__evaluate__ = []
 
     def with_std_random_init_params(self):
-        self.__runner__.add_item(std_random_init_params())
+        self.__init_param__ = std_random_init_params()
         return self
 
-    def with_train_data(self, method, file_list, batch_size=None, **kwargs):
+    def with_train_data(self, method, file_list=None, batch_size=None,
+                        **kwargs):
         if batch_size is None:
             batch_size = self.__network__.optimize_graph().batch_size
 
-        self.__train_data__ = BasicTrainerDataProvider(
+        if file_list is None:
+            file_list = [None]
+
+        self.__train_data__ = BasicPaddleTrainerDataProvider(
             network=self.__network__,
             method=method,
             file_list=file_list,
@@ -77,8 +82,15 @@ def with_observer(self, on_batch_end=None, on_pass_end=None):
         return self
 
     def build(self):
+        if self.__init_param__ is None:
+            self.with_std_random_init_params()
+        self.__runner__.add_item(self.__init_param__)
         self.__runner__.add_item(self.__train_data__)
+        if self.__updater__ is None:
+            self.with_std_local_updater()
         self.__runner__.add_item(self.__updater__)
+        if self.__gradient_machine__ is None:
+            self.with_std_gradient_machine_ops()
         self.__runner__.add_item(self.__gradient_machine__)
         for each in self.__evaluate__:
             self.__runner__.add_item(each)

paddle/py_paddle/trainer/builder.py

@@ -1,51 +1,41 @@
 from .. import DataProviderConverter
 import random
 
-__all__ = ['DataProvider', 'NaiveDataProvider']
+__all__ = ['DataProvider', 'NaiveMemPooledDataProvider', 'NaiveDataProvider']
 
 
 class DataProvider(object):
     __slots__ = [
-        '__init__', 'reset', 'next', '__provider__', '__converter__',
+        '__init__', 'reset', 'next', '__method__', '__converter__',
         '__batch_size__', '__should_shuffle__'
     ]
 
-    def __init__(self, provider, input_types, batch_size, should_shuffle=True):
-        self.__provider__ = provider
+    def __init__(self, method, input_types, batch_size, should_shuffle=True):
+        self.__method__ = method
         self.__converter__ = DataProviderConverter(input_types)
         self.__batch_size__ = batch_size
-        if self.__provider__.should_shuffle is None:
-            self.__provider__.should_shuffle = should_shuffle
+        self.__should_shuffle__ = should_shuffle
 
     def reset(self):
         raise NotImplemented()
 
     def next(self):
         raise NotImplemented()
 
-    def __should_shuffle__(self):
-        return self.__provider__.should_shuffle
-
 
-class NaiveDataProvider(DataProvider):
-    def __init__(self, provider, input_types, batch_size, should_shuffle=True):
-        super(NaiveDataProvider, self).__init__(
-            provider=provider,
+class NaiveMemPooledDataProvider(DataProvider):
+    def __init__(self, method, input_types, batch_size, should_shuffle):
+        super(NaiveMemPooledDataProvider, self).__init__(
+            method=method,
             input_types=input_types,
             batch_size=batch_size,
             should_shuffle=should_shuffle)
         self.__pool__ = []
         self.__idx__ = 0
 
     def reset(self):
-        def __to_pool__():
-            for filename in self.__provider__.file_list:
-                for item in self.__provider__.generator(self.__provider__,
-                                                        filename):
-                    yield item
-
-        self.__pool__ = list(__to_pool__())
-        if self.__should_shuffle__():
+        self.__pool__ = list(self.__method__())
+        if self.__should_shuffle__:
             random.shuffle(self.__pool__)
 
         self.__idx__ = 0
@@ -58,3 +48,17 @@ def next(self):
             return self.__converter__(self.__pool__[begin:end]), end - begin
         else:
             raise StopIteration
+
+
+class NaiveDataProvider(NaiveMemPooledDataProvider):
+    def __init__(self, provider, input_types, batch_size, should_shuffle=True):
+        def __to_pool__():
+            for filename in provider.file_list:
+                for item in provider.generator(provider, filename):
+                    yield item
+
+        super(NaiveDataProvider, self).__init__(
+            method=__to_pool__,
+            input_types=input_types,
+            batch_size=batch_size,
+            should_shuffle=should_shuffle)

paddle/py_paddle/trainer/data_providers.py

@@ -9,8 +9,8 @@
     'BasicLocalParameterUpdaterOps', 'BasicLocalParameterUpdater', 'Counter',
     'BatchEvaluate', 'PassEvaluate', 'BasicGradientMachineTrainOps',
     'BasicGradientMachineTestOps', 'InheritGradientMachineUpdater',
-    'TestOnPassEnd', 'BasicTrainerDataProvider', 'BasicDataProviderOps',
-    'BasicTestDataProvider', 'SaveParamsOnPassEnd', 'BaseObserveItem'
+    'TestOnPassEnd', 'BasicPaddleTrainerDataProvider', 'BasicDataProviderOps',
+    'BasicPaddleTestDataProvider', 'SaveParamsOnPassEnd', 'BaseObserveItem'
 ]
 
 
@@ -330,8 +330,7 @@ def on_batch_begin(self, next_callback):
 def data_provider_creator(is_train):
     class __cls__(BasicDataProviderOps):
         def __init__(self, network, method, file_list, batch_size, **kwargs):
-            BasicDataProviderOps.__init__(self)
-            assert isinstance(network, NetworkConfig)
+            super(__cls__, self).__init__()
             self.__dataprovider__ = method(
                 file_list=file_list,
                 input_order=network.input_order(),
@@ -355,16 +354,16 @@ def initialize(self, context, next_callback):
     return __cls__
 
 
-BasicTrainerDataProvider = data_provider_creator(True)
-BasicTestDataProvider = data_provider_creator(False)
+BasicPaddleTrainerDataProvider = data_provider_creator(True)
+BasicPaddleTestDataProvider = data_provider_creator(False)
 
 
 class TestOnPassEnd(RunnerItem):
     def __init__(self, **kwargs):
         RunnerItem.__init__(self)
         self.__test_runner__ = Runner()
         self.__test_runner__.add_item(InheritGradientMachineUpdater())
-        self.__test_runner__.add_item(BasicTestDataProvider(**kwargs))
+        self.__test_runner__.add_item(BasicPaddleTestDataProvider(**kwargs))
         self.__test_runner__.add_item(BasicGradientMachineTestOps())
         self.__test_runner__.add_item(PassEvaluate(prefix='Test: '))
 

paddle/py_paddle/trainer/items.py

@@ -1,5 +1,6 @@
 from paddle.trainer_config_helpers import *
 from paddle.trainer_config_helpers import inputs as ipts
+import paddle.trainer.PyDataProvider2 as dp2
 
 __all__ = ['NetworkConfig', 'network']
 
@@ -52,6 +53,9 @@ def optimize_graph(self):
         """
         raise NotImplemented()
 
+    def provider(self, **kwargs):
+        return dp2.provider(input_types=self.input_types(), **kwargs)
+
 
 def network(inputs, **opt_kwargs):
     """