CVSROOT: /web/www
Module name: www
Changes by: Waclaw Jacek <wwj> 11/09/18 21:36:18
Modified files:
server/source/linc: linc.py
Log message:
Updating source code of LINC
CVSWeb URLs:
http://web.cvs.savannah.gnu.org/viewcvs/www/server/source/linc/linc.py?cvsroot=www&r1=1.1&r2=1.2
Patches:
Index: linc.py
===================================================================
RCS file: /web/www/www/server/source/linc/linc.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -b -r1.1 -r1.2
--- linc.py 14 Mar 2011 18:44:23 -0000 1.1
+++ linc.py 18 Sep 2011 21:34:56 -0000 1.2
@@ -15,28 +15,32 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-# ---------------------------------------------------------------------
-#
-# Command format: "python checklinks.py <local_directory_containing_website>
<corresponding_address_on_web_server>"
# defines
+BASE_DIRECTORY = '/home/w/wwj/www-repo/'
+REMOTE_BASE_DIRECTORY = 'http://www.gnu.org/'
+
+ADDITIONAL_HTTP_HEADERS = 'User-Agent: LINC/alpha\r\nAccept: text/html,
text/plain, audio/mod, image/*, application/msword, application/pdf,
application/postscript, text/sgml, video/mpeg, */*;q=0.01\r\n' # end every
header with "\r\n"
DELAY_BETWEEN_CHECKS = 1 # In seconds. Set to 0 to disable delay between
checks of different links.
DELAY_BETWEEN_RETRIES = 10 # In seconds. Used when a link fails before
re-checking it. Set to 0 to disable delays.
-NUMBER_OF_ATTEMPTS = 3 # Set to 1 for the program to check links just once,
without retrying in case of failure. 0 makes the program not even check the
link at all, so it is not the most recommended of values.
+FORWARDS_TO_FOLLOW = 5 # How many forwards should be followed.
+NUMBER_OF_ATTEMPTS = 3 # Number of times to check a link for error. If an
attempt is successful, the link is no longer checked during that program run.
REPORT_FILE_NAME = 'reports-temp/broken_links' # Path to the file to which the
errors will be reported.
SEPARATE_TRANSLATION_REPORTS = True # If you set this to True, reports for
translations will be saved into "REPORT_FILE_NAME.lang" instead of in the main
report file.
SKIP_TRANSLATION_FILES = False # If you set this to True, files with
translations will be skipped.
SOCKET_TIMEOUT = 20 # After what time to give up with trying to retrieve a
website.
-EXCLUDED_DIRECTORIES_REGEXP = '(wwwes$|wwwin$|education/fr|software/[^/]+$)' #
Matching directories will not be entered to check their files or subdirectories.
-EXCLUDED_FILENAMES_REGEXP = '\.po$' # Matching files will be ignored.
+# regexp-related defines
+
+EXCLUDED_DIRECTORIES_REGEXP =
'^(japan|wwwes|wwwin|education/fr|software/[^/]+)$' # Matching directories will
not be entered to check their files or subdirectories.
+FILENAMES_TO_CHECK_REGEXP = '\.html$' # Only matching files will be checked.
FTP_LINK_REGEXP = 'ftp://(?P<hostname>[^/:]+)(:(?P<port>[0-9]+))?'
HTTP_ERROR_HEADER = '^HTTP/1\.1 (?P<http_error_code>403|404) ' # What to treat
as a HTTP error header.
+HTTP_FORWARD_HEADER = '^HTTP/1\.1 (301 Moved Permanently|302 Found)$'
HTTP_LINK_REGEXP =
'http://(?P<hostname>[^/:]+)(:(?P<port>[0-9]+))?(?P<resource>/[^#]*)?'
-LINK_REGEXP = '<a( .+?)? href="([^mailto:].+?)"( .+?)?>'
-LINK_REGEXP_GROUP = 1 # Number of the group that matters in the above regexp.
+HTTP_NEW_LOCATION_HEADER = '^Location: (?P<new_location>.+)$'
+LINK_REGEXP = '<a( .+?)? href="(?P<link>[^mailto:].+?)"( .+?)?>'
TRANSLATION_REGEXP = '\.(?P<langcode>[a-z]{2}|[a-z]{2}-[a-z]{2})\.[^.]+$'
# libraries
@@ -76,7 +80,10 @@
socketfd.close()
return None
-def get_http_link_error( link ):
+def get_http_link_error( link, forwarded_from = None ): # forwarded_from is
either None or a list
+ if forwarded_from == None:
+ forwarded_from = []
+
connection_data = re.search( HTTP_LINK_REGEXP, link )
if connection_data:
hostname = connection_data.group( 'hostname' )
@@ -97,7 +104,7 @@
if resource == None:
resource = '/'
- socketfd.send( 'GET ' + resource + ' HTTP/1.1\r\nHost: ' +
hostname + '\r\n\r\n' )
+ socketfd.send( 'GET ' + resource + ' HTTP/1.1\r\nHost: ' +
hostname + '\r\n' + ADDITIONAL_HTTP_HEADERS + '\r\n' )
webpage = socket_read( socketfd )
if webpage == None:
@@ -112,11 +119,44 @@
header_lines = webpage[ : end_of_headers_pos ]
header_lines = header_lines.split( '\r\n' )
- for header_line in header_lines:
- match = re.search( HTTP_ERROR_HEADER, header_line )
+
+ # search for errors
+ match = regexp_search_list( HTTP_ERROR_HEADER, header_lines )
if match:
http_error_code = match.group(
'http_error_code' )
return 'http error ' + http_error_code + '
returned by server'
+
+ # look for forwards
+ match = regexp_search_list( HTTP_FORWARD_HEADER, header_lines )
+ if match:
+ if len( forwarded_from ) < FORWARDS_TO_FOLLOW: # if we
haven't been forwarded too many times yet...
+ match = regexp_search_list(
HTTP_NEW_LOCATION_HEADER, header_lines )
+ if match:
+ forwarded_from.append( link )
+ new_location = match.group(
'new_location' )
+ if new_location in forwarded_from:
+ return 'forward loop!'
+ else:
+ return get_http_link_error(
new_location, forwarded_from )
+ else: # we've been forwarded too many times, sorry.
+ return 'too many forwards (over ' + str( len(
forwarded_from ) ) + ')'
+
+ return None
+
+def is_match_inside_comment_block( regexp_match ):
+ haystack = regexp_match.string
+ match_pos = regexp_match.start()
+ comment_block_start = haystack.rfind( '<!--', 0, match_pos )
+ comment_block_end = haystack.rfind( '-->', 0, match_pos )
+ if comment_block_start > comment_block_end:
+ return True
+ return False
+
+def regexp_search_list( regexp, the_list ):
+ for list_element in the_list:
+ match = re.search( regexp, list_element )
+ if match:
+ return match
return None
def search_directory_for_files( base_directory, directory ):
@@ -129,7 +169,7 @@
search_directory_for_files( base_directory,
relative_path_to_element )
else:
- if re.search( EXCLUDED_FILENAMES_REGEXP, element_name ):
+ if not re.search( FILENAMES_TO_CHECK_REGEXP,
element_name ):
continue
if ( SKIP_TRANSLATION_FILES == True ) and re.search(
TRANSLATION_REGEXP, element_name ):
@@ -162,13 +202,8 @@
### OK, main program below.
-# check if a path has been provided
-if len( sys.argv ) < 3:
- print 'Please run the program with the following arguments: the
directory which should be checked and its corresponding URL on the live site
(so eg.: ./linc.py gnucvs/software/ http://www.gnu.org/software/)'
- sys.exit( 1 )
-
-base_directory = sys.argv[1]
-remote_base_directory = sys.argv[2]
+base_directory = BASE_DIRECTORY
+remote_base_directory = REMOTE_BASE_DIRECTORY
if remote_base_directory[-1] != '/':
remote_base_directory += '/'
@@ -176,7 +211,7 @@
# `cd` to this path
if not os.path.isdir( base_directory ):
- print 'Selected path isn\'t a directory (or doesn\'t exist at all).'
+ print 'Entered base directory isn\'t a directory (or doesn\'t exist at
all).'
sys.exit( 1 )
# list files
@@ -192,8 +227,8 @@
file_contents = fd.read()
fd.close()
- for match in re.findall( LINK_REGEXP, file_contents ):
- link = match[LINK_REGEXP_GROUP]
+ for match in re.finditer( LINK_REGEXP, file_contents ):
+ link = match.group( 'link' )
line_number = -1
split_file_contents = file_contents.split( '\n' )
@@ -203,7 +238,7 @@
line_number = checked_line_number
break
- link_container = { 'filename': file_to_check, 'line_number':
line_number, 'link': link }
+ link_container = { 'filename': file_to_check, 'line_number':
line_number, 'link': link, 'is_link_inside_comment_block':
is_match_inside_comment_block( match ) }
links_to_check.append( link_container )
# check links
@@ -223,6 +258,7 @@
filename = link_container['filename']
line_number = link_container['line_number']
link = link_container['link']
+ is_link_inside_comment_block =
link_container['is_link_inside_comment_block']
link_type = None
@@ -278,6 +314,9 @@
already_checked_links.append( { 'link': link, 'error':
link_error } )
if link_error != None:
+ if is_link_inside_comment_block:
+ link_error += ' (link commented out)'
+
if not SKIP_TRANSLATION_FILES:
match = re.search( TRANSLATION_REGEXP, filename )
if match and SEPARATE_TRANSLATION_REPORTS:
