http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/databricks_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/databricks_hook.html b/_modules/airflow/contrib/hooks/databricks_hook.html index 0d21a4a..0493901 100644 --- a/_modules/airflow/contrib/hooks/databricks_hook.html +++ b/_modules/airflow/contrib/hooks/databricks_hook.html @@ -185,6 +185,7 @@ <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span> <span class="kn">from</span> <span class="nn">requests</span> <span class="k">import</span> <span class="n">exceptions</span> <span class="k">as</span> <span class="n">requests_exceptions</span> <span class="kn">from</span> <span class="nn">requests.auth</span> <span class="k">import</span> <span class="n">AuthBase</span> +<span class="kn">from</span> <span class="nn">time</span> <span class="k">import</span> <span class="n">sleep</span> <span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span> @@ -193,6 +194,9 @@ <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> <span class="kn">import</span> <span class="nn">urlparse</span> +<span class="n">RESTART_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"POST"</span><span class="p">,</span> <span class="s2">"api/2.0/clusters/restart"</span><span class="p">)</span> +<span class="n">START_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"POST"</span><span class="p">,</span> <span class="s2">"api/2.0/clusters/start"</span><span class="p">)</span> +<span class="n">TERMINATE_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"POST"</span><span class="p">,</span> <span class="s2">"api/2.0/clusters/delete"</span><span class="p">)</span> <span class="n">SUBMIT_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'POST'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/submit'</span><span class="p">)</span> <span class="n">GET_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'GET'</span><span class="p">,</span> <span class="s1">'api/2.0/jobs/runs/get'</span><span class="p">)</span> @@ -208,7 +212,8 @@ <span class="bp">self</span><span class="p">,</span> <span class="n">databricks_conn_id</span><span class="o">=</span><span class="s1">'databricks_default'</span><span class="p">,</span> <span class="n">timeout_seconds</span><span class="o">=</span><span class="mi">180</span><span class="p">,</span> - <span class="n">retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">):</span> + <span class="n">retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> + <span class="n">retry_delay</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> :param databricks_conn_id: The name of the databricks connection to use.</span> <span class="sd"> :type databricks_conn_id: string</span> @@ -218,12 +223,17 @@ <span class="sd"> :param retry_limit: The number of times to retry the connection in case of</span> <span class="sd"> service outages.</span> <span class="sd"> :type retry_limit: int</span> +<span class="sd"> :param retry_delay: The number of seconds to wait between retries (it</span> +<span class="sd"> might be a floating point number).</span> +<span class="sd"> :type retry_delay: float</span> <span class="sd"> """</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span> <span class="o">=</span> <span class="n">databricks_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">databricks_conn_id</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span> <span class="o">=</span> <span class="n">timeout_seconds</span> - <span class="k">assert</span> <span class="n">retry_limit</span> <span class="o">>=</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'Retry limit must be greater than equal to 1'</span> + <span class="k">if</span> <span class="n">retry_limit</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span> + <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Retry limit must be greater than equal to 1'</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span> <span class="o">=</span> <span class="n">retry_limit</span> + <span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span> <span class="o">=</span> <span class="n">retry_delay</span> <span class="k">def</span> <span class="nf">_parse_host</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">host</span><span class="p">):</span> <span class="sd">"""</span> @@ -278,7 +288,8 @@ <span class="k">else</span><span class="p">:</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Unexpected HTTP Method: '</span> <span class="o">+</span> <span class="n">method</span><span class="p">)</span> - <span class="k">for</span> <span class="n">attempt_num</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span> <span class="o">+</span> <span class="mi">1</span><span class="p">):</span> + <span class="n">attempt_num</span> <span class="o">=</span> <span class="mi">1</span> + <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> <span class="k">try</span><span class="p">:</span> <span class="n">response</span> <span class="o">=</span> <span class="n">request_func</span><span class="p">(</span> <span class="n">url</span><span class="p">,</span> @@ -286,21 +297,29 @@ <span class="n">auth</span><span class="o">=</span><span class="n">auth</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">USER_AGENT_HEADER</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span><span class="p">)</span> - <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="n">requests</span><span class="o">.</span><span class="n">codes</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> - <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> - <span class="k">else</span><span class="p">:</span> + <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span> + <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> + <span class="k">except</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">RequestException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">_retryable_error</span><span class="p">(</span><span class="n">e</span><span class="p">):</span> <span class="c1"># In this case, the user probably made a mistake.</span> <span class="c1"># Don't retry.</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Response: </span><span class="si">{0}</span><span class="s1">, Status Code: </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> - <span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">))</span> - <span class="k">except</span> <span class="p">(</span><span class="n">requests_exceptions</span><span class="o">.</span><span class="n">ConnectionError</span><span class="p">,</span> - <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">Timeout</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span> - <span class="s1">'Attempt </span><span class="si">%s</span><span class="s1"> API Request to Databricks failed with reason: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> - <span class="n">attempt_num</span><span class="p">,</span> <span class="n">e</span> - <span class="p">)</span> - <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">((</span><span class="s1">'API requests to Databricks failed </span><span class="si">{}</span><span class="s1"> times. '</span> <span class="o">+</span> - <span class="s1">'Giving up.'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">))</span> + <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">))</span> + + <span class="bp">self</span><span class="o">.</span><span class="n">_log_request_error</span><span class="p">(</span><span class="n">attempt_num</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span> + + <span class="k">if</span> <span class="n">attempt_num</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">:</span> + <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">((</span><span class="s1">'API requests to Databricks failed </span><span class="si">{}</span><span class="s1"> times. '</span> <span class="o">+</span> + <span class="s1">'Giving up.'</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">))</span> + + <span class="n">attempt_num</span> <span class="o">+=</span> <span class="mi">1</span> + <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">_log_request_error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attempt_num</span><span class="p">,</span> <span class="n">error</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span> + <span class="s1">'Attempt </span><span class="si">%s</span><span class="s1"> API Request to Databricks failed with reason: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> + <span class="n">attempt_num</span><span class="p">,</span> <span class="n">error</span> + <span class="p">)</span> <div class="viewcode-block" id="DatabricksHook.submit_run"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook.submit_run">[docs]</a> <span class="k">def</span> <span class="nf">submit_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> <span class="sd">"""</span> @@ -331,7 +350,22 @@ <span class="k">def</span> <span class="nf">cancel_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span> <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'run_id'</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">CANCEL_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">CANCEL_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">restart_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">RESTART_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">start_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">START_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">terminate_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">TERMINATE_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span></div> + + +<span class="k">def</span> <span class="nf">_retryable_error</span><span class="p">(</span><span class="n">exception</span><span class="p">):</span> + <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">exception</span><span class="p">,</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">ConnectionError</span><span class="p">)</span> \ + <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">exception</span><span class="p">,</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">Timeout</span><span class="p">)</span> \ + <span class="ow">or</span> <span class="n">exception</span><span class="o">.</span><span class="n">response</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">exception</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">>=</span> <span class="mi">500</span> <span class="n">RUN_LIFE_CYCLE_STATES</span> <span class="o">=</span> <span class="p">[</span> @@ -417,20 +451,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> +
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/datadog_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/datadog_hook.html b/_modules/airflow/contrib/hooks/datadog_hook.html index bb0a05f..cfa1ad2 100644 --- a/_modules/airflow/contrib/hooks/datadog_hook.html +++ b/_modules/airflow/contrib/hooks/datadog_hook.html @@ -331,20 +331,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/datastore_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/datastore_hook.html b/_modules/airflow/contrib/hooks/datastore_hook.html index 29aa16f..5868383 100644 --- a/_modules/airflow/contrib/hooks/datastore_hook.html +++ b/_modules/airflow/contrib/hooks/datastore_hook.html @@ -395,20 +395,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/discord_webhook_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/discord_webhook_hook.html b/_modules/airflow/contrib/hooks/discord_webhook_hook.html index b82d631..6622022 100644 --- a/_modules/airflow/contrib/hooks/discord_webhook_hook.html +++ b/_modules/airflow/contrib/hooks/discord_webhook_hook.html @@ -329,20 +329,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/emr_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/emr_hook.html b/_modules/airflow/contrib/hooks/emr_hook.html index 7aa2a43..e2e6873 100644 --- a/_modules/airflow/contrib/hooks/emr_hook.html +++ b/_modules/airflow/contrib/hooks/emr_hook.html @@ -212,20 +212,7 @@ <span class="n">config</span> <span class="o">=</span> <span class="n">emr_conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> <span class="n">config</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">job_flow_overrides</span><span class="p">)</span> - <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">run_job_flow</span><span class="p">(</span> - <span class="n">Name</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Name'</span><span class="p">),</span> - <span class="n">LogUri</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'LogUri'</span><span class="p">),</span> - <span class="n">ReleaseLabel</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ReleaseLabel'</span><span class="p">),</span> - <span class="n">Instances</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Instances'</span><span class="p">),</span> - <span class="n">Steps</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Steps'</span><span class="p">,</span> <span class="p">[]),</span> - <span class="n">BootstrapActions</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'BootstrapActions'</span><span class="p">,</span> <span class="p">[]),</span> - <span class="n">Applications</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Applications'</span><span class="p">),</span> - <span class="n">Configurations</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Configurations'</span><span class="p">,</span> <span class="p">[]),</span> - <span class="n">VisibleToAllUsers</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'VisibleToAllUsers'</span><span class="p">),</span> - <span class="n">JobFlowRole</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'JobFlowRole'</span><span class="p">),</span> - <span class="n">ServiceRole</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ServiceRole'</span><span class="p">),</span> - <span class="n">Tags</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Tags'</span><span class="p">)</span> - <span class="p">)</span> + <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">run_job_flow</span><span class="p">(</span><span class="o">**</span><span class="n">config</span><span class="p">)</span> <span class="k">return</span> <span class="n">response</span></div></div> </pre></div> @@ -258,20 +245,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/fs_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/fs_hook.html b/_modules/airflow/contrib/hooks/fs_hook.html index 7b01482..ef59c77 100644 --- a/_modules/airflow/contrib/hooks/fs_hook.html +++ b/_modules/airflow/contrib/hooks/fs_hook.html @@ -235,20 +235,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/ftp_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/ftp_hook.html b/_modules/airflow/contrib/hooks/ftp_hook.html index 88879d9..3b25ec7 100644 --- a/_modules/airflow/contrib/hooks/ftp_hook.html +++ b/_modules/airflow/contrib/hooks/ftp_hook.html @@ -309,7 +309,11 @@ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="n">conn</span><span class="o">.</span><span class="n">rmd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div> -<div class="viewcode-block" id="FTPHook.retrieve_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.retrieve_file">[docs]</a> <span class="k">def</span> <span class="nf">retrieve_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">,</span> <span class="n">local_full_path_or_buffer</span><span class="p">):</span> +<div class="viewcode-block" id="FTPHook.retrieve_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.retrieve_file">[docs]</a> <span class="k">def</span> <span class="nf">retrieve_file</span><span class="p">(</span> + <span class="bp">self</span><span class="p">,</span> + <span class="n">remote_full_path</span><span class="p">,</span> + <span class="n">local_full_path_or_buffer</span><span class="p">,</span> + <span class="n">callback</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Transfers the remote file to a local location.</span> @@ -322,23 +326,59 @@ <span class="sd"> :param local_full_path_or_buffer: full path to the local file or a</span> <span class="sd"> file-like buffer</span> <span class="sd"> :type local_full_path_or_buffer: str or file-like buffer</span> +<span class="sd"> :param callback: callback which is called each time a block of data</span> +<span class="sd"> is read. if you do not use a callback, these blocks will be written</span> +<span class="sd"> to the file or buffer passed in. if you do pass in a callback, note</span> +<span class="sd"> that writing to a file or buffer will need to be handled inside the</span> +<span class="sd"> callback.</span> +<span class="sd"> [default: output_handle.write()]</span> +<span class="sd"> :type callback: callable</span> + +<span class="sd"> Example::</span> +<span class="sd"> hook = FTPHook(ftp_conn_id='my_conn')</span> + +<span class="sd"> remote_path = '/path/to/remote/file'</span> +<span class="sd"> local_path = '/path/to/local/file'</span> + +<span class="sd"> # with a custom callback (in this case displaying progress on each read)</span> +<span class="sd"> def print_progress(percent_progress):</span> +<span class="sd"> self.log.info('Percent Downloaded: %s%%' % percent_progress)</span> + +<span class="sd"> total_downloaded = 0</span> +<span class="sd"> total_file_size = hook.get_size(remote_path)</span> +<span class="sd"> output_handle = open(local_path, 'wb')</span> +<span class="sd"> def write_to_file_with_progress(data):</span> +<span class="sd"> total_downloaded += len(data)</span> +<span class="sd"> output_handle.write(data)</span> +<span class="sd"> percent_progress = (total_downloaded / total_file_size) * 100</span> +<span class="sd"> print_progress(percent_progress)</span> +<span class="sd"> hook.retrieve_file(remote_path, None, callback=write_to_file_with_progress)</span> + +<span class="sd"> # without a custom callback data is written to the local_path</span> +<span class="sd"> hook.retrieve_file(remote_path, local_path)</span> <span class="sd"> """</span> <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="n">is_path</span> <span class="o">=</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="n">basestring</span><span class="p">)</span> - <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span> - <span class="n">output_handle</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span> + <span class="c1"># without a callback, default to writing to a user-provided file or</span> + <span class="c1"># file-like buffer</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">callback</span><span class="p">:</span> + <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span> + <span class="n">output_handle</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">local_full_path_or_buffer</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">output_handle</span> <span class="o">=</span> <span class="n">local_full_path_or_buffer</span> + <span class="n">callback</span> <span class="o">=</span> <span class="n">output_handle</span><span class="o">.</span><span class="n">write</span> <span class="k">else</span><span class="p">:</span> - <span class="n">output_handle</span> <span class="o">=</span> <span class="n">local_full_path_or_buffer</span> + <span class="n">output_handle</span> <span class="o">=</span> <span class="kc">None</span> <span class="n">remote_path</span><span class="p">,</span> <span class="n">remote_file_name</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">remote_full_path</span><span class="p">)</span> <span class="n">conn</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">remote_path</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Retrieving file from FTP: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">)</span> - <span class="n">conn</span><span class="o">.</span><span class="n">retrbinary</span><span class="p">(</span><span class="s1">'RETR </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">remote_file_name</span><span class="p">,</span> <span class="n">output_handle</span><span class="o">.</span><span class="n">write</span><span class="p">)</span> + <span class="n">conn</span><span class="o">.</span><span class="n">retrbinary</span><span class="p">(</span><span class="s1">'RETR </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">remote_file_name</span><span class="p">,</span> <span class="n">callback</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Finished retrieving file from FTP: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">)</span> - <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span> + <span class="k">if</span> <span class="n">is_path</span> <span class="ow">and</span> <span class="n">output_handle</span><span class="p">:</span> <span class="n">output_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div> <div class="viewcode-block" id="FTPHook.store_file"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.store_file">[docs]</a> <span class="k">def</span> <span class="nf">store_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_full_path</span><span class="p">,</span> <span class="n">local_full_path_or_buffer</span><span class="p">):</span> @@ -390,7 +430,13 @@ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">)</span></div> - <span class="k">def</span> <span class="nf">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> +<div class="viewcode-block" id="FTPHook.get_mod_time"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.get_mod_time">[docs]</a> <span class="k">def</span> <span class="nf">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Returns a datetime object representing the last time the file was modified</span> + +<span class="sd"> :param path: remote file path</span> +<span class="sd"> :type path: string</span> +<span class="sd"> """</span> <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="n">ftp_mdtm</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">sendcmd</span><span class="p">(</span><span class="s1">'MDTM '</span> <span class="o">+</span> <span class="n">path</span><span class="p">)</span> <span class="n">time_val</span> <span class="o">=</span> <span class="n">ftp_mdtm</span><span class="p">[</span><span class="mi">4</span><span class="p">:]</span> @@ -400,6 +446,16 @@ <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">time_val</span><span class="p">,</span> <span class="s1">'%Y%m</span><span class="si">%d</span><span class="s1">%H%M%S'</span><span class="p">)</span></div> +<div class="viewcode-block" id="FTPHook.get_size"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPHook.get_size">[docs]</a> <span class="k">def</span> <span class="nf">get_size</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Returns the size of a file (in bytes)</span> + +<span class="sd"> :param path: remote file path</span> +<span class="sd"> :type path: string</span> +<span class="sd"> """</span> + <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> + <span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div></div> + <div class="viewcode-block" id="FTPSHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ftp_hook.FTPSHook">[docs]</a><span class="k">class</span> <span class="nc">FTPSHook</span><span class="p">(</span><span class="n">FTPHook</span><span class="p">):</span> @@ -448,20 +504,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/gcp_api_base_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/gcp_api_base_hook.html b/_modules/airflow/contrib/hooks/gcp_api_base_hook.html index 78b729d..22ddd66 100644 --- a/_modules/airflow/contrib/hooks/gcp_api_base_hook.html +++ b/_modules/airflow/contrib/hooks/gcp_api_base_hook.html @@ -248,7 +248,7 @@ <span class="k">elif</span> <span class="n">key_path</span><span class="p">:</span> <span class="c1"># Get credentials from a JSON file.</span> <span class="k">if</span> <span class="n">key_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'.json'</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Getting connection using a JSON key file.'</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Getting connection using JSON key file </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">key_path</span><span class="p">)</span> <span class="n">credentials</span> <span class="o">=</span> <span class="p">(</span> <span class="n">google</span><span class="o">.</span><span class="n">oauth2</span><span class="o">.</span><span class="n">service_account</span><span class="o">.</span><span class="n">Credentials</span><span class="o">.</span><span class="n">from_service_account_file</span><span class="p">(</span> <span class="n">key_path</span><span class="p">,</span> <span class="n">scopes</span><span class="o">=</span><span class="n">scopes</span><span class="p">)</span> @@ -341,20 +341,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/gcp_container_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/gcp_container_hook.html b/_modules/airflow/contrib/hooks/gcp_container_hook.html index 4cb7114..983c362 100644 --- a/_modules/airflow/contrib/hooks/gcp_container_hook.html +++ b/_modules/airflow/contrib/hooks/gcp_container_hook.html @@ -208,6 +208,7 @@ <span class="k">def</span> <span class="nf">_dict_to_proto</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">py_dict</span><span class="p">,</span> <span class="n">proto</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Converts a python dictionary to the proto supplied</span> + <span class="sd"> :param py_dict: The dictionary to convert</span> <span class="sd"> :type py_dict: dict</span> <span class="sd"> :param proto: The proto object to merge with dictionary</span> @@ -223,6 +224,7 @@ <span class="sd">"""</span> <span class="sd"> Given an operation, continuously fetches the status from Google Cloud until either</span> <span class="sd"> completion or an error occurring</span> + <span class="sd"> :param operation: The Operation to wait for</span> <span class="sd"> :type operation: A google.cloud.container_V1.gapic.enums.Operator</span> <span class="sd"> :return: A new, updated operation fetched from Google Cloud</span> @@ -243,6 +245,7 @@ <div class="viewcode-block" id="GKEClusterHook.get_operation"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.get_operation">[docs]</a> <span class="k">def</span> <span class="nf">get_operation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Fetches the operation from Google Cloud</span> + <span class="sd"> :param operation_name: Name of operation to fetch</span> <span class="sd"> :type operation_name: str</span> <span class="sd"> :return: The new, updated operation from Google Cloud</span> @@ -355,6 +358,7 @@ <div class="viewcode-block" id="GKEClusterHook.get_cluster"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.get_cluster">[docs]</a> <span class="k">def</span> <span class="nf">get_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">retry</span><span class="o">=</span><span class="n">DEFAULT</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">DEFAULT</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Gets details of specified cluster</span> + <span class="sd"> :param name: The name of the cluster to retrieve</span> <span class="sd"> :type name: str</span> <span class="sd"> :param retry: A retry object used to retry requests. If None is specified,</span> @@ -406,20 +410,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> +
