Move to beam-user@. I think you can pass a lambda as table reference, is that what you are looking for?
def run <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=42?gsn=run&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run> (argv <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=42?gsn=argv&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.argv> =None): """run pipeline.""" parser <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=44?gsn=parser&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.parser> = argparse.ArgumentParser() _ <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=45?gsn=_&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run._>, pipeline_args <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=45?gsn=pipeline_args&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.pipeline_args> = parser <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=44> .parse_known_args(argv <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=42>) pipeline_options <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=46?gsn=pipeline_options&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.pipeline_options> = PipelineOptions(pipeline_args <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=45> ).view_as(Options <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=17>) pipeline_options <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=46> .view_as(StandardOptions).streaming <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=47?gsn=streaming&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.pipeline_options.view_as.streaming> = True pipeline_options <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=46> .view_as(SetupOptions).save_main_session <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=48?gsn=save_main_session&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.pipeline_options.view_as.save_main_session> = True p <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=49?gsn=p&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.p> = beam <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=10> .Pipeline(options=pipeline_options <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=46>) bq_table <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=51?gsn=bq_table&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.bq_table> = '{}.{}'.format(pipeline_options <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=46>.dataset, pipeline_options <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=46>.table) rows = ( p | 'Read' >> beam.io.ReadFromPubSub(subscription=pipeline_options.input_subscription) | 'Filter' >> beam.Filter(filter_int) | beam <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=10> .Map(lambda x: { 'partition': str(int(x) // 20), 'string_field': x })) def table_ref <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=67?gsn=table_ref&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.table_ref> (value <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=67?gsn=value&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run.table_ref.value>): return pipeline_options.view_as( GoogleCloudOptions ).project + ':' + bq_table <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=51> + '_partition_' + value <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=67> ['partition'] _ <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;bpv=1;bpt=1;rcl=366162493;l=72?gsn=_&gs=kythe%3A%2F%2Fgoogle3%3Flang%3Dpython%3Fpath%3Dthird_party%2Fpy%2Fapache_beam%2Fintegration%2Fbig_query_streaming.py%23module.run._> = ( rows | 'WriteParitionTable' >> beam <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=10> .io <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/io/__init__.py;rcl=366162493;l=0>.WriteToBigQuery( table=table_ref <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=67>, schema='string_field:STRING,' 'partition:STRING')) p <https://source.corp.google.com/piper///depot/google3/third_party/py/apache_beam/integration/big_query_streaming.py;rcl=366162493;l=49> .run() On Fri, Apr 2, 2021 at 8:31 AM Gopinath Kanniyappan <gopin...@gmail.com> wrote: > Hi Team, > > I need some support to implement dynamic destination for WriteToBigquery > method of python version in Steaming mode insert. Can you please provide > some ideas or reference or sample for this ? > > Thanks > Gopi > >
