import os import math import datetime import xml.etree.ElementTree as ET from xml.dom import minidom # For pretty printing XML # --- Configuration --- SOURCE_HTML_DIR = r"./jobs" OUTPUT_DIR = r"./" BASE_URL = "https://walmart.uscareers.co.com/jobs/" MAX_URLS_PER_SITEMAP = 5000 SITEMAP_FILENAME_PREFIX = "sitemap_jobs" SITEMAP_INDEX_FILENAME = "sitemap_index.xml" INCLUDE_CHANGEFREQ_PRIORITY = True DEFAULT_CHANGEFREQ = "daily" DEFAULT_PRIORITY = "0.8" # --- NEW: Configuration for Homepage URL --- INCLUDE_HOMEPAGE = True # Set to True to add the base URL to the sitemap HOMEPAGE_CHANGEFREQ = "daily" # Homepage is usually important and updated often HOMEPAGE_PRIORITY = "1.0" # Priority for homepage should be the highest (1.0) # --- Configuration to force current date for lastmod --- FORCE_CURRENT_DATE_FOR_LASTMOD = True # --- Configuration to only include files modified today --- SITEMAP_ONLY_TODAY_URLS = False # --- End Configuration --- # --- Helper Functions --- def find_html_files_generator(directory): """ A generator that yields HTML file paths one by one using os.scandir for memory efficiency. """ print(f"Scanning directory: {directory}") try: for entry in os.scandir(directory): if entry.is_file() and entry.name.lower().endswith(".html"): yield entry.path except FileNotFoundError: print(f"Error: Source directory not found: {directory}") except Exception as e: print(f"Error scanning directory {directory}: {e}") def format_datetime_w3c(dt_object): """Formats a datetime object into W3C Datetime format (UTC).""" return dt_object.strftime('%Y-%m-%dT%H:%M:%SZ') def prettify_xml(elem): """Returns a pretty-printed XML string for the Element.""" rough_string = ET.tostring(elem, 'utf-8') reparsed = minidom.parseString(rough_string) return reparsed.toprettyxml(indent=" ", encoding='utf-8') # --- Main Logic --- def generate_sitemaps(): if not os.path.exists(OUTPUT_DIR): print(f"Output directory '{OUTPUT_DIR}' not found. Creating it...") try: os.makedirs(OUTPUT_DIR) print(f"Created output directory: {OUTPUT_DIR}") except OSError as e: print(f"Error creating output directory '{OUTPUT_DIR}': {e}") return current_utc_datetime = datetime.datetime.now(datetime.timezone.utc) current_w3c_time_str = format_datetime_w3c(current_utc_datetime) current_utc_date = current_utc_datetime.date() if FORCE_CURRENT_DATE_FOR_LASTMOD: print(f"WARNING: Forcing all dates to current generation time: {current_w3c_time_str}") if SITEMAP_ONLY_TODAY_URLS: print(f"WARNING: SITEMAP_ONLY_TODAY_URLS is enabled. Only including files modified on {current_utc_date} (UTC).") html_file_generator = find_html_files_generator(SOURCE_HTML_DIR) generated_sitemap_files = [] total_files_processed = 0 total_files_skipped = 0 sitemap_num = 1 # NEW: Derive homepage URL from BASE_URL homepage_url = "/".join(BASE_URL.split("/")[:3]) + "/" while True: urlset_attrs = {"xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9"} urlset = ET.Element("urlset", attrib=urlset_attrs) file_count_in_batch = 0 # --- NEW: Add homepage to the very first sitemap file --- if sitemap_num == 1 and INCLUDE_HOMEPAGE: print("\nAdding homepage to the sitemap...") url_element = ET.SubElement(urlset, "url") ET.SubElement(url_element, "loc").text = homepage_url ET.SubElement(url_element, "lastmod").text = current_w3c_time_str # Use current time ET.SubElement(url_element, "changefreq").text = HOMEPAGE_CHANGEFREQ ET.SubElement(url_element, "priority").text = HOMEPAGE_PRIORITY file_count_in_batch += 1 # We don't increment total_files_processed here to keep the file count accurate print(f"Homepage <{homepage_url}> added with priority {HOMEPAGE_PRIORITY}.") # --- End of new section --- # Check if this is the first file being added to a new sitemap first_file_in_batch = True for html_filepath in html_file_generator: try: if SITEMAP_ONLY_TODAY_URLS: last_mod_timestamp = os.path.getmtime(html_filepath) file_mod_date = datetime.datetime.utcfromtimestamp(last_mod_timestamp).date() if file_mod_date != current_utc_date: total_files_skipped += 1 continue if first_file_in_batch: # Print header only when starting a new file print(f"\nGenerating {SITEMAP_FILENAME_PREFIX}{sitemap_num}.xml...") first_file_in_batch = False filename_with_ext = os.path.basename(html_filepath) loc_url = f"{BASE_URL}{filename_with_ext}" if FORCE_CURRENT_DATE_FOR_LASTMOD: last_mod_w3c = current_w3c_time_str else: last_mod_timestamp = os.path.getmtime(html_filepath) dt_object = datetime.datetime.utcfromtimestamp(last_mod_timestamp) last_mod_w3c = format_datetime_w3c(dt_object) url_element = ET.SubElement(urlset, "url") ET.SubElement(url_element, "loc").text = loc_url ET.SubElement(url_element, "lastmod").text = last_mod_w3c if INCLUDE_CHANGEFREQ_PRIORITY: ET.SubElement(url_element, "changefreq").text = DEFAULT_CHANGEFREQ ET.SubElement(url_element, "priority").text = DEFAULT_PRIORITY file_count_in_batch += 1 total_files_processed += 1 except Exception as e: print(f" Warning: Could not process file '{html_filepath}' for sitemap entry: {e}") if file_count_in_batch >= MAX_URLS_PER_SITEMAP: break # Check if any URLs (including homepage) were added to the batch if file_count_in_batch > 0: sitemap_filename = f"{SITEMAP_FILENAME_PREFIX}{sitemap_num}.xml" sitemap_filepath = os.path.join(OUTPUT_DIR, sitemap_filename) try: xml_bytes = prettify_xml(urlset) with open(sitemap_filepath, "wb") as f: f.write(xml_bytes) # Adjust count message if homepage was the only entry url_count_message = f"{file_count_in_batch} URL" if file_count_in_batch > 1: url_count_message += "s" if sitemap_num == 1 and INCLUDE_HOMEPAGE: url_count_message += " (including homepage)" print(f"Successfully created '{sitemap_filepath}' with {url_count_message}.") generated_sitemap_files.append(sitemap_filename) sitemap_num += 1 except Exception as e: print(f"Error writing sitemap file '{sitemap_filepath}': {e}") else: break # The rest of the script for generating the sitemap index remains the same print(f"\nTotal HTML files processed: {total_files_processed}") if SITEMAP_ONLY_TODAY_URLS: print(f"Total HTML files skipped (not modified today): {total_files_skipped}") if not generated_sitemap_files: print("\nNo individual sitemaps were generated. Cannot create sitemap index.") if INCLUDE_HOMEPAGE and total_files_processed == 0: print("Note: The homepage was configured to be included, but no sitemap was generated because no other HTML files were found.") return # --- Generate Sitemap Index --- index_filepath = os.path.join(OUTPUT_DIR, SITEMAP_INDEX_FILENAME) print(f"\nGenerating sitemap index file: {index_filepath}...") sitemapindex = ET.Element("sitemapindex", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9") domain_base = "/".join(BASE_URL.split("/")[:3]) # More robust way to get base domain for sitemap_file_name in generated_sitemap_files: try: sitemap_loc_url = f"{domain_base}/{sitemap_file_name}" sitemap_last_mod_w3c_for_index_entry = "" if FORCE_CURRENT_DATE_FOR_LASTMOD: sitemap_last_mod_w3c_for_index_entry = current_w3c_time_str else: individual_sitemap_full_path = os.path.join(OUTPUT_DIR, sitemap_file_name) sitemap_last_mod_timestamp = os.path.getmtime(individual_sitemap_full_path) dt_object = datetime.datetime.utcfromtimestamp(sitemap_last_mod_timestamp) sitemap_last_mod_w3c_for_index_entry = format_datetime_w3c(dt_object) sitemap_element = ET.SubElement(sitemapindex, "sitemap") ET.SubElement(sitemap_element, "loc").text = sitemap_loc_url ET.SubElement(sitemap_element, "lastmod").text = sitemap_last_mod_w3c_for_index_entry except Exception as e: print(f" Warning: Could not process sitemap index entry for '{sitemap_file_name}': {e}") if len(generated_sitemap_files) > 0: try: xml_bytes_index = prettify_xml(sitemapindex) with open(index_filepath, "wb") as f: f.write(xml_bytes_index) print(f"Successfully created sitemap index '{index_filepath}'.") except Exception as e: print(f"Error writing sitemap index file '{index_filepath}': {e}") else: print("Sitemap index not created as no individual sitemaps were generated.") print("\nSitemap generation process finished.") print("IMPORTANT: Upload ALL generated .xml files to the root directory of your website.") print(f"Then submit ONLY the sitemap index URL to search engines: {domain_base}/{SITEMAP_INDEX_FILENAME}") if __name__ == "__main__": if not os.path.isdir(SOURCE_HTML_DIR): print("="*50) print("!!! PLEASE EDIT THE SCRIPT CONFIGURATION !!!") print(f"The SOURCE_HTML_DIR ('{SOURCE_HTML_DIR}') does not seem to be a valid directory.") print("Please update it to the correct path where your HTML files are located.") print("Also check 'OUTPUT_DIR' and 'BASE_URL'.") print("="*50) elif not BASE_URL.startswith("http"): print("="*50) print("!!! PLEASE EDIT THE SCRIPT CONFIGURATION !!!") print(f"The BASE_URL ('{BASE_URL}') does not look like a valid absolute URL.") print("It should start with http:// or https://") print("="*50) else: generate_sitemaps()