John, just wanted to thank you for your help. I've also pasted in the file, a sample 
data file and the output to this message, in case anyone's following along in the 
archives.

You're suggestions to improve time were amazing. I went from analyzing an 84,000 line 
log file in 30 minutes to just seconds.

Thank you again for all your help.

-Kevin Zembower
=================================================
[EMAIL PROTECTED]:/opt/analog/logdata/db$ cat listqueries3.pl
#!/usr/bin/perl

use warnings;
use strict;

my $debug = 1;

while (<>) {
   next if /^#/; #Skip it if it's one of the comments at the beginning of the file.
   my ($date, $time, $source, undef, undef, $host, $FQDN, $method, $file, $query, 
undef) = split;
   next unless $file =~ /dbtwpcgi\.exe/ and $query =~ /&TN=popline&/i and $query =~ 
/QF0/;
   my $type = $query =~ /QI2/? 'A': 'B';
     
   my %hash = (abstract =>'', author =>'', engtitle =>'', docno =>'', keywordsmajor 
=>'', journaltitle =>'', languages =>'', popreporttopic =>'', refereed =>'', year 
=>'');
   #This is the more elaborate version, suggested by John Krahn
   $hash{ lc $2 } = $3 while $query =~ 
/QF(\d+)=\b(abstract|author|engtitle|docno|keywordsmajor|journaltitle|languages|popreporttopic|refereed|year)\b(?=.*?&QI\1=([^&]*))/ig;

   my $outstring = join("\t", $type, $date, $time, @hash{ qw( abstract author engtitle 
docno keywordsmajor journaltitle languages popreporttopic refereed year)}) . "\n";
   print translate($outstring);
}# while there are more lines in the input file   

sub translate() {
   $_ = $_[0];
   s/%22/\"/g;
   s/%2C/,/g;
   s/%20/ /g;
   s#%2F#/#g;
   s/%3D/=/g;
   s/%3B/;/g;
   s/%26/&/g;
   s/%0D//g;
   s/%0A//g;
   s/\+/ /g;
   s/%29/)/g;
   s/%28/(/g;
   s/%27/'/g;
   s/%2b/+/g;
   s/%7C/|/g;
   s/%3A/:/g;
   #Debbie request all boolean logical words and sumbols be replaced with '|'
   s/\band\b/|/ig;
   s/\bor\b/|/ig;
   tr!&/!|!;
   $_;
   }
[EMAIL PROTECTED]:/opt/analog/logdata/db$ cat x
2004-03-28 00:38:31 d7.facsmf.utexas.edu - W3SVC1 DB db.jhuccp.org GET 
/dbtw-wpd/exec/dbtwpcgi.exe 
XC=%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&BU=http%3A%2F%2Fdb.jhuccp.org%2Fpopinform%2Fbasic.html&QB0=AND&QF0=Abstract+%7C+KeywordsMajor+%7C+KeywordsMinor+%7C+Notes+%7C+EngTitle+%7C+TT+%7C+FREAb+%7C+SPAAb&QI0=China%0D%0A&QB1=AND&QF1=Author+%7C+CN&QI1=&MR=10&TN=popline&RF=ShortRecordDisplay&DF=LongRecordDisplay&DL=1&RL=1&NP=0&AC=QBE_QUERY&x=37&y=4
 200 0 21248 814 19391 80 HTTP/1.1 
Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+.NET+CLR+1.0.3705) - 
http://db.jhuccp.org/popinform/basic.html
2004-03-28 00:38:31 d7.facsmf.utexas.edu - W3SVC1 DB db.jhuccp.org GET 
/dbtw-wpd/jscript/dbtw-r-main.js - 200 0 6256 647 172 80 HTTP/1.1 
Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+.NET+CLR+1.0.3705) - 
http://db.jhuccp.org/dbtw-wpd/exec/dbtwpcgi.exe?XC=%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&BU=http%3A%2F%2Fdb.jhuccp.org%2Fpopinform%2Fbasic.html&QB0=AND&QF0=Abstract+%7C+KeywordsMajor+%7C+KeywordsMinor+%7C+Notes+%7C+EngTitle+%7C+TT+%7C+FREAb+%7C+SPAAb&QI0=China%0D%0A&QB1=AND&QF1=Author+%7C+CN&QI1=&MR=10&TN=popline&RF=ShortRecordDisplay&DF=LongRecordDisplay&DL=1&RL=1&NP=0&AC=QBE_QUERY&x=37&y=4
2004-03-28 00:38:31 d7.facsmf.utexas.edu - W3SVC1 DB db.jhuccp.org GET 
/popinform/res-style.css - 200 0 2212 639 63 80 HTTP/1.1 
Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+.NET+CLR+1.0.3705) - 
http://db.jhuccp.org/dbtw-wpd/exec/dbtwpcgi.exe?XC=%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&BU=http%3A%2F%2Fdb.jhuccp.org%2Fpopinform%2Fbasic.html&QB0=AND&QF0=Abstract+%7C+KeywordsMajor+%7C+KeywordsMinor+%7C+Notes+%7C+EngTitle+%7C+TT+%7C+FREAb+%7C+SPAAb&QI0=China%0D%0A&QB1=AND&QF1=Author+%7C+CN&QI1=&MR=10&TN=popline&RF=ShortRecordDisplay&DF=LongRecordDisplay&DL=1&RL=1&NP=0&AC=QBE_QUERY&x=37&y=4
2004-03-30 14:59:24 218.11.80.208 - W3SVC1 DB db.jhuccp.org GET 
/dbtw-wpd/jscript/dbtw-r-main.js - 200 0 6256 444 2078 80 HTTP/1.1 
Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+NT+5.0) - 
http://db.jhuccp.org/dbtw-wpd/exec/dbtwpcgi.exe?BU=http%3A%2F%2Fdb.jhuccp.org%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&QF0=Country&QI0=China&TN=Condoms&AC=QBE_QUERY&MR=30&RF=ShortRecordDisplay&NP=3&DF=results_long
2004-03-30 14:59:24 218.11.80.208 - W3SVC1 DB db.jhuccp.org GET 
/dbtw-wpd/exec/dbtwpcgi.exe 
BU=http%3A%2F%2Fdb.jhuccp.org%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&QF0=Country&QI0=China&TN=Condoms&AC=QBE_QUERY&MR=30&RF=ShortRecordDisplay&NP=3&DF=results_long
 200 0 33987 588 5109 80 HTTP/1.1 Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+NT+5.0) 
- http://condoms.jhuccp.org/asia.html
2004-03-31 13:28:05 proxy.unam.na - W3SVC1 DB db.jhuccp.org GET 
/dbtw-wpd/exec/dbtwpcgi.exe 
XC=%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&BU=http%3A%2F%2Fdb.jhuccp.org%2Fpopinform%2Fbasic.html&QB0=AND&QF0=Abstract+%7C+KeywordsMajor+%7C+KeywordsMinor+%7C+Notes+%7C+EngTitle+%7C+TT+%7C+FREAb+%7C+SPAAb&QI0=child+labour+AND++literacy+NOT+China+&QB1=AND&QF1=Author+%7C+CN&QI1=&MR=10&TN=popline&RF=ShortRecordDisplay&DF=LongRecordDisplay&DL=1&RL=1&NP=0&AC=QBE_QUERY&x=22&y=15
 200 0 751 821 1484 80 HTTP/1.0 Mozilla/4.75+[en]+(Win98;+U) - 
http://db.jhuccp.org/popinform/basic.html
2004-03-31 19:41:46 172.16.4.160 - W3SVC1 DB db.jhuccp.org GET 
/dbtw-wpd/jscript/dbtw-r-main.js - 200 0 6256 874 15 80 HTTP/1.1 
Mozilla/4.0+(compatible;+MSIE+5.5;+Windows+NT+4.0) - 
http://db.jhuccp.org/dbtw-wpd/exec/dbtwpcgi.exe?XC=%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&BU=http%3A%2F%2Fdb.jhuccp.org%2Fpopinform%2Fexpert.html&MR=10&QB10=AND&QF10=OnlineDate&QI10=&QF0=Author+%7C+CN&QI0=&QB0=AND&QF1=EngTitle+%7C+TT&QI1=&QB1=AND&QF5=JournalTitle+%7C+MI+%7C+Imprint+%7C+Series&QI5=&QB5=AND&QF9=Refereed&QF3=Abstract%7CKeywordsMajor%7CKeywordsMinor%7CNotes%7CEngTitle%7CTT%7CFREAb%7CSPAAb&QI3=IUDs+%26+China&QB3=AND&QF6=Year&QI6=&QB6=AND&QF4=Languages&QI4=&QB4=AND&QF2=KeywordsMajor%7CKeywordsMinor&QI2=&QB2=AND&QF7=DocNo&QI7=&QB7=AND&TN=popline&RF=ShortRecordDisplay&DF=LongRecordDisplay&DL=1&RL=1&NP=0&AC=QBE_QUERY&x=21&y=6
2004-03-31 19:41:46 172.16.4.160 - W3SVC1 DB db.jhuccp.org GET 
/dbtw-wpd/exec/dbtwpcgi.exe 
XC=%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&BU=http%3A%2F%2Fdb..jhuccp.org%2Fpopinform%2Fexpert.html&MR=10&QB10=AND&QF10=OnlineDate&QI10=&QF0=Author+%7C+CN&QI0=&QB0=AND&QF1=EngTitle+%7C+TT&QI1=&QB1=AND&QF5=JournalTitle+%7C+MI+%7C+Imprint+%7C+Series&QI5=&QB5=AND&QF9=Refereed&QF3=Abstract%7CKeywordsMajor%7CKeywordsMinor%7CNotes%7CEngTitle%7CTT%7CFREAb%7CSPAAb&QI3=IUDs+%26+China&QB3=AND&QF6=Year&QI6=&QB6=AND&QF4=Languages&QI4=&QB4=AND&QF2=KeywordsMajor%7CKeywordsMinor&QI2=&QB2=AND&QF7=DocNo&QI7=&QB7=AND&TN=popline&RF=ShortRecordDisplay&DF=LongRecordDisplay&DL=1&RL=1&NP=0&AC=QBE_QUERY&x=21&y=6
 200 0 21063 1025 2891 80 HTTP/1.1 Mozilla/4.0+(compatible;+MSIE+5.5;+Windows+NT+4.0) 
- http://db.jhuccp.org/popinform/expert.html
2004-03-31 19:41:46 172.16.4.160 - W3SVC1 DB db.jhuccp.org GET /popinform/infoCart.js 
- 304 0 142 953 0 80 HTTP/1.1 Mozilla/4.0+(compatible;+MSIE+5.5;+Windows+NT+4.0) - 
http://db.jhuccp.org/dbtw-wpd/exec/dbtwpcgi.exe?XC=%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&BU=http%3A%2F%2Fdb.jhuccp.org%2Fpopinform%2Fexpert.html&MR=10&QB10=AND&QF10=OnlineDate&QI10=&QF0=Author+%7C+CN&QI0=&QB0=AND&QF1=EngTitle+%7C+TT&QI1=&QB1=AND&QF5=JournalTitle+%7C+MI+%7C+Imprint+%7C+Series&QI5=&QB5=AND&QF9=Refereed&QF3=Abstract%7CKeywordsMajor%7CKeywordsMinor%7CNotes%7CEngTitle%7CTT%7CFREAb%7CSPAAb&QI3=IUDs+%26+China&QB3=AND&QF6=Year&QI6=&QB6=AND&QF4=Languages&QI4=&QB4=AND&QF2=KeywordsMajor%7CKeywordsMinor&QI2=&QB2=AND&QF7=DocNo&QI7=&QB7=AND&TN=popline&RF=ShortRecordDisplay&DF=LongRecordDisplay&DL=1&RL=1&NP=0&AC=QBE_QUERY&x=21&y=6
2004-03-31 19:41:46 172.16.4.160 - W3SVC1 DB db.jhuccp.org GET 
/popinform/res-style.css - 200 0 2212 866 0 80 HTTP/1.1 
Mozilla/4.0+(compatible;+MSIE+5..5;+Windows+NT+4.0) - 
http://db.jhuccp.org/dbtw-wpd/exec/dbtwpcgi.exe?XC=%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&BU=http%3A%2F%2Fdb.jhuccp.org%2Fpopinform%2Fexpert.html&MR=10&QB10=AND&QF10=OnlineDate&QI10=&QF0=Author+%7C+CN&QI0=&QB0=AND&QF1=EngTitle+%7C+TT&QI1=&QB1=AND&QF5=JournalTitle+%7C+MI+%7C+Imprint+%7C+Series&QI5=&QB5=AND&QF9=Refereed&QF3=Abstract%7CKeywordsMajor%7CKeywordsMinor%7CNotes%7CEngTitle%7CTT%7CFREAb%7CSPAAb&QI3=IUDs+%26+China&QB3=AND&QF6=Year&QI6=&QB6=AND&QF4=Languages&QI4=&QB4=AND&QF2=KeywordsMajor%7CKeywordsMinor&QI2=&QB2=AND&QF7=DocNo&QI7=&QB7=AND&TN=popline&RF=ShortRecordDisplay&DF=LongRecordDisplay&DL=1&RL=1&NP=0&AC=QBE_QUERY&x=21&y=6
2004-04-30 23:56:26 dhcp-103-107.sphtm.tulane.edu - W3SVC1 DB db.jhuccp.org GET 
/dbtw-wpd/exec/dbtwpcgi.exe 
BU=http%3A%2F%2Fdb.jhuccp.org%2Fdbtw-wpd%2Fexec%2Fdbtwpcgi.exe&QF0=KeywordsMajor%&QF1=Year&QI0=HIV_Infections*%26women%&QI1=2001%3A2004&TN=Popline&AC=QBE_QUERY&MR=30&DL=1&&RL=1&&RF=ShortRecordDisplay&DF=LongRecordDisplay
 200 0 42970 594 1610 80 HTTP/1.1 Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1) - 
http://db.jhuccp.org/popinform/basic.html
[EMAIL PROTECTED]:/opt/analog/logdata/db$ ./listqueries3.pl x
main::translate() called too early to check prototype at ./listqueries3.pl line 19.
B       2004-03-28      00:38:31        China
B       2004-03-31      13:28:05        child labour |  literacy NOT China 
A       2004-03-31      19:41:46        IUDs | China
B       2004-04-30      23:56:26                                        
HIV_Infections*|women%         2001:2004
[EMAIL PROTECTED]:/opt/analog/logdata/db$



--
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
<http://learn.perl.org/> <http://learn.perl.org/first-response>


Reply via email to