I am using python 2.7.6 to write a web crawling program with web2py as the user interface. A large amount of UPCs (used to find the items on the websites) are stored in an SQLite database, which is then displayed on the main html page via the JQuery Datatables plugin. The user can then select one of the upcs via the datatables display which uses ajax to upload the data to the server (this works perfectly).
Currently, I am trying to set it up to handle multiple datatables
selections. I believe I have gotten the multi-selection capabilities
working on the view side, but when I run a debug, only the data from the
first selected UPC is shown on the server.
Thanks for your time,
Philip
Relevant default.py section and all relevant files are attached:
form_labels = {
'sku_wb': XML('SKU for WBShop '),
'zip_code': XML('Zip Code (5 digits or none) '),
'apo': XML('Outside 50 States '),
'manual': XML('Manual UPC (Ignores above title selection) ')
}
def getMediaDataTable():
return TABLE(
THEAD(TR(
*[TH(media_table_dict[i_label]['label']) for i_label in
media_table_names]
)),
_id='title_select_table', _class='display', _cellspacing="0", _width
="100%"
)
def getRequestForm():
media_list_datatable = getMediaDataTable()
new_request_form = FORM(
media_list_datatable, BR(), BR(), BR(),
TABLE(
# TR( TD(form_labels['zip_code']), TD(DIV(zip_code_input,
_class='zip_code_div')) ),
# TR( TD(form_labels['apo']), TD(DIV(apo_input,
_class='apo_div')) ),
TR( TD(form_labels['manual']), TD(DIV(manual_input,
_class='manual_input_div')) ),
BR(),
TR( TD(request_submit, _colspan=2, _align='center') )
)
)
return new_request_form
# GLOBAL vars
zip_code_input = INPUT(_name='zip_code', _value='')
apo_input = INPUT(_name='apo', _type='checkbox', _value='Outside 50 States')
manual_input = INPUT(_name='manual_input', _value='')
request_submit = INPUT(_type='submit', _value="Perform Request")
def pricetool():
request_form = getRequestForm()
is_init_session = session.is_initialized is None
if is_init_session:
session.is_initialized = True
session.media_list_total = settings.media_list_total
session.media_list_selection=None
session.visible_columns=['title', 'upc']
upc_keyword_entry = request.vars['upc']
selected_entry = session.media_list_selection
manual_input_entry = request.vars['manual_input']
is_keyword_request = upc_keyword_entry is not None and
upc_keyword_entry != ''
is_selected_entry = not is_init_session and selected_entry is not None
and "row" in selected_entry
is_manual_input = manual_input_entry is not None and manual_input_entry
!= ''
is_form_accepted = request_form.accepts(request, session)
is_selection = is_form_accepted and is_selected_entry
is_manual = is_form_accepted and is_manual_input
if is_manual:
upc_request = manual_input_entry
title_request = upc_request
elif is_selection:
selected_record = media_db(media_table.DT_RowId == selected_entry).
select().first()
print (selected_record)
is_record_found = selected_record is not None
if is_record_found:
upc_request = selected_record.upc
title_request = selected_record.title
else:
is_form_accepted = False
if is_form_accepted or is_keyword_request:
zip_entry = request.vars['zip_code']
apo_entry = request.vars['apo']
tab_title_entry = request.vars['tab_title']
is_zip = zip_entry is not None and len(zip_entry) == 5
is_apo_checked = apo_entry == form_labels['apo']
is_apo_true = apo_entry == True
is_tab_title = tab_title_entry is not None and tab_title_entry != ''
zip_code = zip_entry if is_zip else None
is_apo = True if is_apo_checked or is_apo_true else None
tab_title = tab_title_entry if is_tab_title else 'PriceTool'
if is_form_accepted:
redirect_dict = dict(
upc=upc_request,
tab_title=title_request
)
if is_zip:
redirect_dict['zip_code']=zip_code
if is_apo:
redirect_dict['apo']=True
redirect_url = URL(f='pricetool',vars=redirect_dict)
redirect(redirect_url)
if is_keyword_request:
session.media_list_selection=None
results_record = results_db(results_table.upc == upc_keyword_entry).
select().first()
results_only_entry = request.vars['results_only']
is_results_only = results_only_entry is not None and
results_only_entry != '0'
if is_results_only:
request_form = DIV(request_form, _hidden='hidden')
return makeOutput(
request_form=request_form,
upc_request=upc_keyword_entry,
results_record=results_record,
zip_request=zip_code,
apo_request=is_apo,
tab_title=tab_title
)
else:
return makeOutput(request_form=request_form)
--
Resources:
- http://web2py.com
- http://web2py.com/book (Documentation)
- http://github.com/web2py/web2py (Source code)
- https://code.google.com/p/web2py/issues/list (Report Issues)
---
You received this message because you are subscribed to the Google Groups
"web2py-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to web2py+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
Title: {{=tab_title}}
{{=tab_title}}
{{=request_form}}
{{=upc_db_link}}
| {{=item_image}} | {{=item_title}} |
{{=results_tables}}
import os
import csv
import datetime
from grabbercreator import GrabberCreator
media_filename = 'storage.media'
media_db = DAL('sqlite://' + media_filename)
media_table_names = [
'DT_RowId', 'title', 'upc', 'studio', 'status', 'versions', 'broadcast',
'release_date', 'msrp', 'genre', 'aspect', 'sound', 'is_released', 'rating'
]
media_table_dict = {
'DT_RowId': {
'label': XML('DT_RowId'),
'type': 'text'
},
'title': {
'label': XML('Title'),
'type': 'text'
},
'upc': {
'label': XML('UPC'),
'type': 'text'
},
'studio': {
'label': XML('Studio'),
'type': 'text'
},
'status': {
'label': XML('Status'),
'type': 'text'
},
'versions': {
'label': XML('Format'),
'type': 'text'
},
'broadcast': {
'label': XML('Broadcast'),
'type': 'text'
},
'release_date': {
'label': XML('Release Date'),
'type': 'text'
},
'msrp': {
'label': XML('MSRP'),
'type': 'text'
},
'genre': {
'label': XML('Genre'),
'type': 'text'
},
'aspect': {
'label': XML('Aspect Ratio'),
'type': 'text'
},
'sound': {
'label': XML('Audio Type'),
'type': 'text'
},
'is_released': {
'label': XML('Is Released'),
'type': 'text'
},
'rating': {
'label': XML('Rating'),
'type': 'text'
}
}
media_table_fields = [
Field(
i_field,
label=media_table_dict[i_field]['label'],
type=media_table_dict[i_field]['type']
)
for i_field in media_table_names
]
media_table_fields += [Field('title_with_upc', compute=lambda row: row['title'] + "|" + row['upc'])]
media_table_fields += [Field('title_orderby', compute=lambda row: row['title'].upper())]
media_db.define_table(
'media_table',
*media_table_fields
)
media_table = media_db.media_table
results_filename = 'storage.results'
results_db = DAL('sqlite://' + results_filename)
results_table_names = ['upc', 'results', 'timestamp']
results_table_dict = {
'upc': {
'label': XML('UPC'),
'type': 'text'
},
'results': {
'label': XML('Historical Results'),
'type': 'json'
},
'timestamp': {
'label': XML('Last Update'),
'type': 'datetime'
}
}
results_table_fields = [
Field(
i_field,
label=results_table_dict[i_field]['label'],
type=results_table_dict[i_field]['type']
)
for i_field in results_table_names
]
results_db.define_table(
'results_table',
*results_table_fields
)
results_table = results_db.results_table
if media_db(media_table.id > 0).isempty():
dvd_list_filename = os.path.join(request.folder, 'static/dvd_csv.txt')
dvd_list_labels = [
'title', 'studio', 'released', 'status', 'sound', 'versions',
'price', 'rating', 'year', 'genre', 'aspect', 'upc', 'dvdyear'
]
with open(dvd_list_filename, mode='rb') as dvd_list_file:
rows = csv.reader(dvd_list_file)
next(rows)
dvd_list_dicts = [{
'DT_RowId': "row_" + str(id_num),
'title': row[dvd_list_labels.index('title')],
'upc': row[dvd_list_labels.index('upc')],
'studio': row[dvd_list_labels.index('studio')],
'status': row[dvd_list_labels.index('status')],
'versions': row[dvd_list_labels.index('versions')],
'broadcast': row[dvd_list_labels.index('year')],
'release_date':
datetime.datetime.strptime(
row[dvd_list_labels.index('dvdyear')],
'%Y-%m-%d %H:%M:%S'
).date().isoformat() if row[dvd_list_labels.index('dvdyear')] != '' else '',
'msrp': row[dvd_list_labels.index('price')],
'genre': row[dvd_list_labels.index('genre')],
'aspect': row[dvd_list_labels.index('aspect')],
'sound': row[dvd_list_labels.index('sound')],
'is_released':
datetime.datetime.strptime(
row[dvd_list_labels.index('released')],
'%Y-%m-%d %H:%M:%S'
).date().isoformat() if row[dvd_list_labels.index('released')] != '' else 'Released',
'rating': row[dvd_list_labels.index('rating')]
} for id_num, row in enumerate(rows)]
media_db.media_table.bulk_insert(dvd_list_dicts)
settings_db = DAL('sqlite://storage.settings')
settings_db.define_table(
'settings_table',
Field('media_list_total', 'integer'),
Field('update_begin_hour', 'integer')
)
settings_table = settings_db.settings_table
is_settings_record = not settings_db(settings_table.id > 0).isempty()
if not is_settings_record:
settings_insert = settings_table.insert()
settings = settings_db(settings_table.id > 0).select().first()
media_list_total = media_db(media_table.id > 0).count()
settings.update_record(media_list_total=media_list_total)
settings.update_record(update_begin_hour=4)
else:
settings = settings_db(settings_table.id > 0).select().first()
collections_db = DAL('sqlite://storage.collections')
collections_db.define_table(
'upc_table',
Field('upc'),
Field('number_of_titles') # titles of the collection this upc composes (multiple seasons, series, etc)
)
collections_db.define_table(
'title_table',
Field('name'), # from user input title
Field('upcs', type='list:reference upc_table'), # comma separated list of participating UPCs for this collection title
)
collections_db.define_table(
'collections_table',
Field('name'),
Field('titles', type='list:reference title_table'), # comma separated list, matches name in title table, user input
)
collections_table = collections_db.collections_table
# stores_db = DAL('sqlite://storage.stores')
# stores_db.define_table(
# 'stores_table',
# Field('name'),
# Field('store_icon', 'upload'),
# Field('platform'), # if set, establishes defaults for below entries
# Field('supplier_country'), # for price currency conversion, just USA and CA for now
# Field('url_address'),
# Field('price_find'), # for prices, titles, images, or web-page redirections
# Field('no_result_find'),
# Field('url_find'),
# Field('exact_inventory_find'),
# Field('exact_inventory_break'),
# Field('inventory_quantity_find'),
# Field('direct_link', 'boolean'),
# Field('is_user_agent', 'boolean'),
# Field('rebate_type'), # establishes defaults by platform, also allows individual settings
# Field('rebate_percent'),
# Field('coupon_type'), # Fixed percent, limit-based terms
# Field('coupon_terms'),
# Field('shipping_type'), # Fixed, limit based terms, or search terms
# Field('shipping_terms'),
# Field('tax_type'), # includes a type that can override local tax rate for state rates given in terms
# Field('tax_terms') # includes inclusions/exclusions
# )
#
# stores_table = stores_db.stores_table
#
# if stores_db(stores_table.id > 0).isempty():
# item_upc = "___TEST___UPC___"
# grabber_creator = GrabberCreator(item_upc, app_dir=os.path.join(request.folder, 'modules/'), is_alt_only=False)
# grabbers = grabber_creator.grabbers_list
# alt_grabbers = grabber_creator.alt_grabbers_list
from pricefinder import PriceFinder
import json
import os
form_labels = {
'sku_wb': XML('SKU for WBShop '),
'zip_code': XML('Zip Code (5 digits or none) '),
'apo': XML('Outside 50 States '),
'manual': XML('Manual UPC (Ignores above title selection) ')
}
def getMediaDataTable():
return TABLE(
THEAD(TR(
*[TH(media_table_dict[i_label]['label']) for i_label in media_table_names]
)),
_id='title_select_table', _class='display', _cellspacing="0", _width="100%"
)
def getRequestForm():
media_list_datatable = getMediaDataTable()
new_request_form = FORM(
media_list_datatable, BR(), BR(), BR(),
TABLE(
# TR( TD(form_labels['zip_code']), TD(DIV(zip_code_input, _class='zip_code_div')) ),
# TR( TD(form_labels['apo']), TD(DIV(apo_input, _class='apo_div')) ),
TR( TD(form_labels['manual']), TD(DIV(manual_input, _class='manual_input_div')) ),
BR(),
TR( TD(request_submit, _colspan=2, _align='center') )
)
)
return new_request_form
# GLOBAL vars
zip_code_input = INPUT(_name='zip_code', _value='')
apo_input = INPUT(_name='apo', _type='checkbox', _value='Outside 50 States')
manual_input = INPUT(_name='manual_input', _value='')
request_submit = INPUT(_type='submit', _value="Perform Request")
def pricetool():
request_form = getRequestForm()
is_init_session = session.is_initialized is None
if is_init_session:
session.is_initialized = True
session.media_list_total = settings.media_list_total
session.media_list_selection=None
session.visible_columns=['title', 'upc']
upc_keyword_entry = request.vars['upc']
selected_entry = session.media_list_selection
manual_input_entry = request.vars['manual_input']
is_keyword_request = upc_keyword_entry is not None and upc_keyword_entry != ''
is_selected_entry = not is_init_session and selected_entry is not None and "row" in selected_entry
is_manual_input = manual_input_entry is not None and manual_input_entry != ''
is_form_accepted = request_form.accepts(request, session)
is_selection = is_form_accepted and is_selected_entry
is_manual = is_form_accepted and is_manual_input
if is_manual:
upc_request = manual_input_entry
title_request = upc_request
elif is_selection:
selected_record = media_db(media_table.DT_RowId == selected_entry).select().first()
print (selected_record)
is_record_found = selected_record is not None
if is_record_found:
upc_request = selected_record.upc
title_request = selected_record.title
else:
is_form_accepted = False
if is_form_accepted or is_keyword_request:
zip_entry = request.vars['zip_code']
apo_entry = request.vars['apo']
tab_title_entry = request.vars['tab_title']
is_zip = zip_entry is not None and len(zip_entry) == 5
is_apo_checked = apo_entry == form_labels['apo']
is_apo_true = apo_entry == True
is_tab_title = tab_title_entry is not None and tab_title_entry != ''
zip_code = zip_entry if is_zip else None
is_apo = True if is_apo_checked or is_apo_true else None
tab_title = tab_title_entry if is_tab_title else 'PriceTool'
if is_form_accepted:
redirect_dict = dict(
upc=upc_request,
tab_title=title_request
)
if is_zip:
redirect_dict['zip_code']=zip_code
if is_apo:
redirect_dict['apo']=True
redirect_url = URL(f='pricetool',vars=redirect_dict)
redirect(redirect_url)
if is_keyword_request:
session.media_list_selection=None
results_record = results_db(results_table.upc == upc_keyword_entry).select().first()
results_only_entry = request.vars['results_only']
is_results_only = results_only_entry is not None and results_only_entry != '0'
if is_results_only:
request_form = DIV(request_form, _hidden='hidden')
return makeOutput(
request_form=request_form,
upc_request=upc_keyword_entry,
results_record=results_record,
zip_request=zip_code,
apo_request=is_apo,
tab_title=tab_title
)
else:
return makeOutput(request_form=request_form)
def selectMediaList(select_string, start_pos = 0, max_rows = 250, orderby='title', order_direction='asc', queryby=['title']):
if select_string != '':
select_strings = select_string.split()
if 'title' in queryby and 'upc' in queryby:
column_to_search = 'title_with_upc'
elif 'title' in queryby:
column_to_search = 'title'
else:
column_to_search = 'upc'
query_db = eval(
'media_table.'
+ column_to_search
+ '.contains(select_strings, all=True)'
)
else:
query_db = media_table
if orderby == 'title':
orderby = 'title_orderby'
if order_direction == 'asc':
orderby_db = eval('media_table.' + orderby)
else:
orderby_db = eval('~media_table.' + orderby)
selected_rows = media_db(query_db).select(
limitby=(start_pos,start_pos+max_rows),
orderby=orderby_db
)
num_rows = media_db(query_db).count()
selected_dict = selected_rows.as_list()
return selected_dict, num_rows
def selectMediaItem():
media_list_selection = str(request.vars['selected']).replace("'","").replace('"',"")
session.media_list_selection=media_list_selection
return settings
def createMediaDataTable():
title_entry = request.vars['search[value]']
if title_entry == 'Media Search':
title_entry = ''
draw_echo = int(request.vars['draw'])
start_pos = int(request.vars['start'])
max_rows = int(request.vars['length'])
order_column = request.vars['order[0][column]']
orderby_name = request.vars['columns[' + order_column + '][data]']
order_direction = request.vars['order[0][dir]']
media_data, media_list_num = selectMediaList(
title_entry,
start_pos=start_pos,
max_rows=max_rows,
orderby=orderby_name,
order_direction=order_direction,
queryby=session.visible_columns
)
media_data_json = json.dumps(dict(
media_data=media_data,
recordsTotal=settings.media_list_total,
recordsFiltered=media_list_num,
draw = draw_echo
))
return media_data_json
def initDataTables():
session.visible_columns=['title', 'upc']
def setColumnVisibility():
searchable_columns = ['title', 'upc']
column = media_table_names[int(request.vars['column'])]
if column in searchable_columns:
state = request.vars['state']
if state == 'true':
visible_columns = session.visible_columns + column
else:
visible_columns = session.visible_columns.remove(column)
session.visible_columns=visible_columns
return settings
def makeOutput(request_form=None, upc_request=None, results_record=None, zip_request=None, apo_request=False, sku_wb=None, tab_title='PriceTool'):
upc_db_link = A(
'Click to add or update UPCs',
_href=URL(upc_db_update)
)
if upc_request is None:
return dict(
request_form=request_form,
upc_title='',
wb_title='',
item_title='',
item_image='',
results_tables='',
budget_table='',
upc_db_link=upc_db_link,
tab_title=tab_title
)
upc_title = "Getting results for UPC: " + upc_request
skus = [sku_wb] if sku_wb is not None else []
if sku_wb is None:
sku_wb = 'SKU not given'
wb_title = "SKU WB:" + sku_wb
is_historical_results = results_record is not None
if is_historical_results:
historical_results = results_record.results
time_historical = results_record.timestamp
time_now = request.now
time_last_update = time_now.replace(hour=settings.update_begin_hour, minute=0, second=0)
is_fresh_results = time_historical > time_last_update
else:
is_fresh_results = False
if is_fresh_results:
item_title = historical_results['item_title']
image_url = historical_results['image_url']
results_dtb = encodeResultsXML(historical_results['results_dtb'])
results_ea = encodeResultsXML(historical_results['results_ea'])
results_international = encodeResultsXML(historical_results['results_international'])
results_exclusions = encodeResultsXML(historical_results['results_exclusions'])
results_budget = historical_results['results_budget']
header_dtb = historical_results['header_dtb']
header_ea = historical_results['header_ea']
header_international = historical_results['header_international']
header_exclusions = historical_results['header_exclusions']
header_budget = historical_results['header_budget']
else:
price_finder = PriceFinder(
item_upc=upc_request,
skus=skus,
zip_code=zip_request,
apo=apo_request,
app_dir=os.path.join(request.folder, 'modules/')
)
price_finder.updateAll()
results_creator = price_finder.results_creator
results_dtb, header_dtb = results_creator.getResultsDTB()
results_ea, header_ea = results_creator.getResultsEA()
results_international, header_international = results_creator.getResultsInternational()
results_exclusions, header_exclusions = results_creator.getResultsExclusions()
results_budget, header_budget = results_creator.getResultsBudget()
item_title = results_creator.product_title
image_url = results_creator.image_url
historical_results = dict()
historical_results['item_title'] = item_title
historical_results['image_url'] = image_url
historical_results['results_dtb'] = results_dtb
historical_results['results_ea'] = results_ea
historical_results['results_international'] = results_international
historical_results['results_exclusions'] = results_exclusions
historical_results['results_budget'] = results_budget
historical_results['header_dtb'] = header_dtb
historical_results['header_ea'] = header_ea
historical_results['header_international'] = header_international
historical_results['header_exclusions'] = header_exclusions
historical_results['header_budget'] = header_budget
if is_historical_results:
results_record.update_record(
upc=upc_request,
results=historical_results,
timestamp=request.now
)
else:
results_table.insert(
upc=upc_request,
results=historical_results,
timestamp=request.now
)
results_db.commit()
if item_title == "":
item_title = "No title results"
if image_url != "":
item_image = IMG(_src=image_url)
else:
item_image = "No image results"
dtb_table = makeTable(
results_dtb,
header_dtb,
table_name="Direct to Buyer",
table_class_name="dtb_table",
td_class_name="dtb_td",
tr_class_names=["odd", "even"]
)
ea_table = makeTable(
results_ea,
header_ea,
table_name="To East Aurora",
table_class_name="ea_table",
td_class_name="ea_td",
tr_class_names=["odd", "even"]
)
international_table = makeTable(
results_international,
header_international,
table_name="International Suppliers",
table_class_name="international_table",
td_class_name="international_td",
tr_class_names=["odd", "even"]
)
exclusions_table = makeTable(
results_exclusions,
header_exclusions,
table_name="Exclusions",
table_class_name="exclusions_table",
td_class_name="exclusions_td",
tr_class_names=["odd", "even"]
)
results_tables = TABLE(
TBODY(
TR(TD(dtb_table),TD(ea_table),TD(international_table)),
TR(TD(exclusions_table, _colspan=3, _align='center'))
),
_class="results_table"
)
budget_table = makeBudgetTable(results_budget, header_budget)
return dict(
request_form=request_form,
upc_title=upc_title,
wb_title=wb_title,
item_title=item_title,
item_image=item_image,
results_tables=results_tables,
budget_table=budget_table,
upc_db_link=upc_db_link,
tab_title=tab_title
)
def encodeResultsXML(
results
):
return [[XML(cell) for cell in row] for row in results]
def makeTable(
results,
header,
table_name="",
table_class_name="",
td_class_name="",
tr_class_names=['']
):
return TABLE(
THEAD(TD(table_name,_class=td_class_name, _colspan=str(len(header))), TR(*TH(header))),
TBODY(*[TR(*TD(rows),_class=tr_class_names[i_row % len(tr_class_names)]) for i_row, rows in enumerate(results)]),
_class="display", _id=table_class_name
)
def makeBudgetTable(results, header):
return TABLE(
TBODY(*[TR(TD(i_header), TD(XML(results[0][i_num]))) for i_num, i_header in enumerate(header)]),
_class = "budgettable"
)
def getFromForm(form_in, field_name):
in_request = form_in.vars.getlist(field_name)
try:
in_request = in_request[0]
except:
in_request = ""
return in_request
def upc_db_update():
upc_grid = SQLFORM.grid(media_table, user_signature=False)
pricetool_link = A(
'Click to return to PriceTool',
_href=URL(pricetool)
)
return locals()
def stores_db_update():
stores_grid = SQLFORM.grid(stores_db.stores_table, user_signature=False)
pricetool_link = A(
'Click to return to PriceTool',
_href=URL(pricetool)
)
return locals()
