From 2ede82f862c8275b7fc399e06c5262e2b6360df8 Mon Sep 17 00:00:00 2001
From: InsonusK <Insonus.K@gmail.com>
Date: Sat, 20 Nov 2021 23:00:50 +0300
Subject: [PATCH 1/5] extend data check

---
 finam/export.py                          |  7 ++++---
 tests/test_FinamTooLongTimeframeError.py | 12 ++++++++++++
 2 files changed, 16 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_FinamTooLongTimeframeError.py

diff --git a/finam/export.py b/finam/export.py
index 42e6bd3..a3fbcd8 100644
--- a/finam/export.py
+++ b/finam/export.py
@@ -257,8 +257,9 @@ class Exporter(object):
     EMPTY_RESULT_NOT_TICKS = '<DATE>;<TIME>;<OPEN>;<HIGH>;<LOW>;<CLOSE>;<VOL>'
     EMPTY_RESULT_TICKS = '<TICKER>;<PER>;<DATE>;<TIME>;<LAST>;<VOL>'
 
-    ERROR_TOO_MUCH_WANTED = (u'Вы запросили данные за слишком '
-                             u'большой временной период')
+    ERROR_TOO_MUCH_WANTED = [(u'Вы запросили данные за слишком '
+                             u'большой временной период'),
+                             "Максимальная глубина данных"]
 
     ERROR_THROTTLING = 'Forbidden: Access is denied'
     ERROR_ALREADY_IN_PROGRESS = u'Система уже обрабатывает Ваш запрос'
@@ -286,7 +287,7 @@ def _postprocess(self, data, timeframe):
         return data
 
     def _sanity_check(self, data):
-        if self.ERROR_TOO_MUCH_WANTED in data:
+        if any(error in data for error in self.ERROR_TOO_MUCH_WANTED):
             raise FinamTooLongTimeframeError
 
         if self.ERROR_THROTTLING in data:
diff --git a/tests/test_FinamTooLongTimeframeError.py b/tests/test_FinamTooLongTimeframeError.py
new file mode 100644
index 0000000..a49eaf3
--- /dev/null
+++ b/tests/test_FinamTooLongTimeframeError.py
@@ -0,0 +1,12 @@
+import unittest
+from finam.export import fetch_url, Exporter
+from finam.exception import FinamTooLongTimeframeError
+
+class FinamTooLongTimeframeErrorTestCase(unittest.TestCase):
+    def test_tooDeep(self):
+        url = "https://export.finam.ru/export9.out?market=5&em=83&token=03AGdBq24TszZ-dPqHt33nEErdKlymyopEkaclaoED4Nm5NP078LpwSMcZApE1B-iL18KsheOiMdb3sVACEi7Ou91RxlZn7yQmfLoS3FgoCx7rSfIHjf-2ab4Ye5xASsdLFzND7PiF7Fuuv3qx0NswSb9V_dL3Q8sH304ZT8CjCg5XZFrVKwIVIErqi63hFli_xsOgUXFHiukCDeG6KtPG9znQzRMDxgDaApBsjGApFh4CAX5RFi40HqXSdNEJ92uMQHXaIrZT75--AzX-PBXFDtM4DFpHGzZKvMGAGc5IlM2WBZZiIIlHvFzCRwRxG2d5CtfWQpfYKN6g3GnH5w2NGf9ApwGqvmwN2AZrjsMYLKyuJwEmI-7O2HS7xj4MuJkvlcdPxhvgH9Om2mhJ-OR-_7CWSuY3H8xvWGwFlf-wDepX7TpxmKHvwucx_yoydfTy61DTwCf_P0Mf&code=EURUSD&apply=0&df=15&mf=7&yf=2019&from=15.08.2019&dt=15&mt=7&yt=2021&to=15.08.2021&p=7&f=EURUSD_190815_210815&e=.txt&cn=EURUSD&dtf=1&tmf=1&MSOR=1&mstime=on&mstimever=1&sep=1&sep2=1&datf=1&at=1"
+        data = fetch_url(url)
+        
+        export = Exporter()
+        with self.assertRaises(FinamTooLongTimeframeError):
+            export._sanity_check(data)
\ No newline at end of file

From 294c15bcbc12efff86440255cea1214f03b9f773 Mon Sep 17 00:00:00 2001
From: InsonusK <Insonus.K@gmail.com>
Date: Sat, 20 Nov 2021 23:41:44 +0300
Subject: [PATCH 2/5] add interval divider

---
 finam/export.py                          | 126 ++++++++++++-----------
 finam/interval.py                        |   3 +-
 tests/test_FinamTooLongTimeframeError.py |  19 +++-
 3 files changed, 87 insertions(+), 61 deletions(-)

diff --git a/finam/export.py b/finam/export.py
index a3fbcd8..cf0c2f9 100644
--- a/finam/export.py
+++ b/finam/export.py
@@ -311,7 +311,8 @@ def download(self,
                  timeframe=Timeframe.DAILY,
                  delay=1,
                  max_in_progress_retries=10,
-                 fill_empty=False):
+                 fill_empty=False,
+                 max_interval_divider = 10):
         items = self._meta.lookup(id_=id_, market=market)
         # i.e. for markets 91, 519, 2
         # id duplicates are feasible, looks like corrupt data on finam
@@ -324,64 +325,73 @@ def download(self,
         if end_date is None:
             end_date = datetime.date.today()
 
-        df = None
-        chunks = split_interval(start_date, end_date, timeframe.value)
-        counter = 0
-        for chunk_start_date, chunk_end_date in chunks:
-            counter += 1
-            logger.info('Processing chunk %d of %d', counter, len(chunks))
-            if counter > 1:
-                logger.info('Sleeping for {} second(s)'.format(delay))
-                time.sleep(delay)
-
-            params = {
-                'p': timeframe.value,
-                'em': id_,
-                'market': market.value,
-                'df': chunk_start_date.day,
-                'mf': chunk_start_date.month - 1,
-                'yf': chunk_start_date.year,
-                'dt': chunk_end_date.day,
-                'mt': chunk_end_date.month - 1,
-                'yt': chunk_end_date.year,
-                'cn': code,
-                'code': code,
-                # I would guess this param denotes 'data format'
-                # that differs for ticks only
-                'datf': 6 if timeframe == Timeframe.TICKS.value else 5,
-                'fsp': 1 if fill_empty else 0
-            }
-
-            url = self._build_url(params)
-            # deliberately not using pd.read_csv's ability to fetch
-            # urls to fully control what's happening
-            retries = 0
-            while True:
-                data = self._fetcher(url)
-                data = self._postprocess(data, timeframe)
-                try:
-                    self._sanity_check(data)
-                except FinamAlreadyInProgressError:
-                    if retries <= max_in_progress_retries:
-                        retries += 1
-                        logger.info('Finam work is in progress, sleeping'
-                                    ' for {} second(s) before retry #{}'
-                                    .format(delay, retries))
+        interval_divider = 1
+        while True:
+            df = None
+            chunks = split_interval(start_date, end_date, timeframe.value, interval_divider)
+            counter = 0
+            try:
+                for chunk_start_date, chunk_end_date in chunks:
+                    counter += 1
+                    logger.info('Processing chunk %d of %d', counter, len(chunks))
+                    if counter > 1:
+                        logger.info('Sleeping for {} second(s)'.format(delay))
                         time.sleep(delay)
-                        continue
-                    else:
-                        raise
-                break
 
-            try:
-                chunk_df = pd.read_csv(StringIO(data), sep=';')
-                chunk_df.sort_index(inplace=True)
-            except ParserError as e:
-                raise FinamParsingError(e)
+                    params = {
+                        'p': timeframe.value,
+                        'em': id_,
+                        'market': market.value,
+                        'df': chunk_start_date.day,
+                        'mf': chunk_start_date.month - 1,
+                        'yf': chunk_start_date.year,
+                        'dt': chunk_end_date.day,
+                        'mt': chunk_end_date.month - 1,
+                        'yt': chunk_end_date.year,
+                        'cn': code,
+                        'code': code,
+                        # I would guess this param denotes 'data format'
+                        # that differs for ticks only
+                        'datf': 6 if timeframe == Timeframe.TICKS.value else 5,
+                        'fsp': 1 if fill_empty else 0
+                    }
+
+                    url = self._build_url(params)
+                    # deliberately not using pd.read_csv's ability to fetch
+                    # urls to fully control what's happening
+                    retries = 0
+                    while True:
+                        data = self._fetcher(url)
+                        data = self._postprocess(data, timeframe)
+                        try:
+                            self._sanity_check(data)
+                        except FinamAlreadyInProgressError:
+                            if retries <= max_in_progress_retries:
+                                retries += 1
+                                logger.info('Finam work is in progress, sleeping'
+                                            ' for {} second(s) before retry #{}'
+                                            .format(delay, retries))
+                                time.sleep(delay)
+                                continue
+                            else:
+                                raise
+                        break
+
+                    try:
+                        chunk_df = pd.read_csv(StringIO(data), sep=';')
+                        chunk_df.sort_index(inplace=True)
+                    except ParserError as e:
+                        raise FinamParsingError(e)
+
+                    if df is None:
+                        df = chunk_df
+                    else:
+                        df = df.append(chunk_df)
 
-            if df is None:
-                df = chunk_df
-            else:
-                df = df.append(chunk_df)
+                return df
+            except FinamTooLongTimeframeError:
+                if interval_divider < max_interval_divider:
+                    interval_divider = interval_divider + 1
+                else:
+                    raise
 
-        return df
diff --git a/finam/interval.py b/finam/interval.py
index 7cd732d..8d4bd35 100644
--- a/finam/interval.py
+++ b/finam/interval.py
@@ -17,12 +17,13 @@
 }
 
 
-def split_interval(start_date, end_date, timeframe):
+def split_interval(start_date, end_date, timeframe, interval_divider = 1):
     if end_date < start_date:
         raise ValueError('start_date must be >= end_date, but got {} and {}'
                          .format(start_date, end_date))
     delta_days = (end_date - start_date).days + 1
     max_days = _MAX_DAYS_PER_TIMEFRAME[timeframe]
+    max_days = max_days // interval_divider
     chunks_count, remainder = divmod(delta_days, max_days)
     if remainder != 0:
         chunks_count += 1
diff --git a/tests/test_FinamTooLongTimeframeError.py b/tests/test_FinamTooLongTimeframeError.py
index a49eaf3..21d8061 100644
--- a/tests/test_FinamTooLongTimeframeError.py
+++ b/tests/test_FinamTooLongTimeframeError.py
@@ -1,6 +1,8 @@
 import unittest
-from finam.export import fetch_url, Exporter
+from finam.export import fetch_url, Exporter, Timeframe
 from finam.exception import FinamTooLongTimeframeError
+from finam.const import Market
+from datetime import date
 
 class FinamTooLongTimeframeErrorTestCase(unittest.TestCase):
     def test_tooDeep(self):
@@ -9,4 +11,17 @@ def test_tooDeep(self):
         
         export = Exporter()
         with self.assertRaises(FinamTooLongTimeframeError):
-            export._sanity_check(data)
\ No newline at end of file
+            export._sanity_check(data)
+
+    def test_if_divide_then_get_data_when_load_tooLongPeriod(self):
+        date_from = date(2020,1,1)
+        date_till = date(2021,1,2)
+        export = Exporter()
+        export.download(83,Market(5),date_from, date_till,Timeframe.MINUTES15)
+    
+    def test_get_exception_when_load_tooLongPeriod(self):
+        date_from = date(2020,1,1)
+        date_till = date(2021,2,2)
+        export = Exporter()
+        with self.assertRaises(FinamTooLongTimeframeError):
+            export.download(83,Market(5),date_from, date_till,Timeframe.MINUTES15,max_interval_divider=1)
\ No newline at end of file

From 79a7f54eb72efb364340d67c280b97e559da6ef0 Mon Sep 17 00:00:00 2001
From: InsonusK <Insonus.K@gmail.com>
Date: Sun, 21 Nov 2021 00:10:59 +0300
Subject: [PATCH 3/5] Change tests

---
 tests/test_FinamTooLongTimeframeError.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/test_FinamTooLongTimeframeError.py b/tests/test_FinamTooLongTimeframeError.py
index 21d8061..b8fc317 100644
--- a/tests/test_FinamTooLongTimeframeError.py
+++ b/tests/test_FinamTooLongTimeframeError.py
@@ -3,14 +3,16 @@
 from finam.exception import FinamTooLongTimeframeError
 from finam.const import Market
 from datetime import date
+from nose.tools import assert_raises
 
-class FinamTooLongTimeframeErrorTestCase(unittest.TestCase):
+class FinamTooLongTimeframeErrorTestCase(object):
     def test_tooDeep(self):
         url = "https://export.finam.ru/export9.out?market=5&em=83&token=03AGdBq24TszZ-dPqHt33nEErdKlymyopEkaclaoED4Nm5NP078LpwSMcZApE1B-iL18KsheOiMdb3sVACEi7Ou91RxlZn7yQmfLoS3FgoCx7rSfIHjf-2ab4Ye5xASsdLFzND7PiF7Fuuv3qx0NswSb9V_dL3Q8sH304ZT8CjCg5XZFrVKwIVIErqi63hFli_xsOgUXFHiukCDeG6KtPG9znQzRMDxgDaApBsjGApFh4CAX5RFi40HqXSdNEJ92uMQHXaIrZT75--AzX-PBXFDtM4DFpHGzZKvMGAGc5IlM2WBZZiIIlHvFzCRwRxG2d5CtfWQpfYKN6g3GnH5w2NGf9ApwGqvmwN2AZrjsMYLKyuJwEmI-7O2HS7xj4MuJkvlcdPxhvgH9Om2mhJ-OR-_7CWSuY3H8xvWGwFlf-wDepX7TpxmKHvwucx_yoydfTy61DTwCf_P0Mf&code=EURUSD&apply=0&df=15&mf=7&yf=2019&from=15.08.2019&dt=15&mt=7&yt=2021&to=15.08.2021&p=7&f=EURUSD_190815_210815&e=.txt&cn=EURUSD&dtf=1&tmf=1&MSOR=1&mstime=on&mstimever=1&sep=1&sep2=1&datf=1&at=1"
         data = fetch_url(url)
         
         export = Exporter()
-        with self.assertRaises(FinamTooLongTimeframeError):
+        
+        with assert_raises(FinamTooLongTimeframeError):
             export._sanity_check(data)
 
     def test_if_divide_then_get_data_when_load_tooLongPeriod(self):
@@ -23,5 +25,5 @@ def test_get_exception_when_load_tooLongPeriod(self):
         date_from = date(2020,1,1)
         date_till = date(2021,2,2)
         export = Exporter()
-        with self.assertRaises(FinamTooLongTimeframeError):
+        with assert_raises(FinamTooLongTimeframeError):
             export.download(83,Market(5),date_from, date_till,Timeframe.MINUTES15,max_interval_divider=1)
\ No newline at end of file

From c7e44fdbaf4f55cb323df3bfb6a32f8a31580041 Mon Sep 17 00:00:00 2001
From: InsonusK <Insonus.K@gmail.com>
Date: Sun, 21 Nov 2021 01:07:18 +0300
Subject: [PATCH 4/5] add delay to exclude error on tryes

---
 finam/export.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/finam/export.py b/finam/export.py
index cf0c2f9..cee1604 100644
--- a/finam/export.py
+++ b/finam/export.py
@@ -392,6 +392,7 @@ def download(self,
             except FinamTooLongTimeframeError:
                 if interval_divider < max_interval_divider:
                     interval_divider = interval_divider + 1
+                    time.sleep(delay)
                 else:
                     raise
 

From 21bc28b656413ee2e79f1cf1abbfcdbdac97de87 Mon Sep 17 00:00:00 2001
From: InsonusK <insonus.k@gmail.com>
Date: Sun, 13 Feb 2022 17:07:52 +0300
Subject: [PATCH 5/5] =?UTF-8?q?=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB?=
 =?UTF-8?q?=20=D1=80=D0=B5=D0=B3=D0=B8=D1=81=D1=82=D1=80=D0=B0=20=D0=BD?=
 =?UTF-8?q?=D0=B5=D0=B7=D0=B0=D0=B2=D0=B8=D1=81=D0=B8=D0=BC=D0=BE=D1=81?=
 =?UTF-8?q?=D1=82=D1=8C=20=D0=B4=D0=BB=D1=8F=20=D0=BF=D1=80=D0=BE=D0=B2?=
 =?UTF-8?q?=D0=B5=D1=80=D0=BA=D0=B8=20=D0=B1=D0=BE=D0=BB=D1=8C=D1=88=D0=B8?=
 =?UTF-8?q?=D1=85=20=D0=B8=D0=BD=D1=82=D0=B5=D1=80=D0=B2=D0=B0=D0=BB=D0=BE?=
 =?UTF-8?q?=D0=B2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 finam/export.py | 9 +++++----
 setup.py        | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/finam/export.py b/finam/export.py
index cee1604..e6793d6 100644
--- a/finam/export.py
+++ b/finam/export.py
@@ -257,9 +257,10 @@ class Exporter(object):
     EMPTY_RESULT_NOT_TICKS = '<DATE>;<TIME>;<OPEN>;<HIGH>;<LOW>;<CLOSE>;<VOL>'
     EMPTY_RESULT_TICKS = '<TICKER>;<PER>;<DATE>;<TIME>;<LAST>;<VOL>'
 
-    ERROR_TOO_MUCH_WANTED = [(u'Вы запросили данные за слишком '
+    ERROR_TOO_MUCH_WANTED = [(u'вы запросили данные за слишком '
                              u'большой временной период'),
-                             "Максимальная глубина данных"]
+                             "максимальная глубина данных",
+                             "делайте несколько запросов"]
 
     ERROR_THROTTLING = 'Forbidden: Access is denied'
     ERROR_ALREADY_IN_PROGRESS = u'Система уже обрабатывает Ваш запрос'
@@ -287,7 +288,7 @@ def _postprocess(self, data, timeframe):
         return data
 
     def _sanity_check(self, data):
-        if any(error in data for error in self.ERROR_TOO_MUCH_WANTED):
+        if any(error in data.lower() for error in self.ERROR_TOO_MUCH_WANTED):
             raise FinamTooLongTimeframeError
 
         if self.ERROR_THROTTLING in data:
@@ -391,7 +392,7 @@ def download(self,
                 return df
             except FinamTooLongTimeframeError:
                 if interval_divider < max_interval_divider:
-                    interval_divider = interval_divider + 1
+                    interval_divider = interval_divider * 2
                     time.sleep(delay)
                 else:
                     raise
diff --git a/setup.py b/setup.py
index e5ac24f..b9395d6 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 # coding: utf8
 from setuptools import setup
 
-VERSION = '5.1.1'
+VERSION = '5.1.2'
 
 long_description = open('README.md').read()