// content.js


var accumulatedData = [];
var ScrapedDetails = [];
window.hasAlreadyScraped = false;


async function scrollToBottom({
   containerSelector = '#search-page-list-container',
   maxScrolls = 8,
   scrollDelay = 1500
} = {}) {
   const container = document.querySelector(containerSelector);


   if (!container) {
       console.error(`❌ Container with selector '${containerSelector}' not found.`);
       return;
   }


   try {
       const h1Element = document.querySelector('#grid-search-results h1');
       if (h1Element) {
           h1Element.click();
           console.log("✅ Clicked h1 element to trigger initial content load.");
           await new Promise(resolve => setTimeout(resolve, 500));
       }
   } catch (error) {
       console.warn("⚠️ Failed to click h1 element:", error);
   }


   let lastScrollTop = -1;
   let scrollAttempts = 0;


   while (scrollAttempts < maxScrolls) {
       const currentScrollTop = container.scrollTop;
       container.scrollBy(0, 600);
       console.log(`🔄 Scroll attempt ${scrollAttempts + 1}`);


       await new Promise(resolve => setTimeout(resolve, scrollDelay));


       if (container.scrollTop === lastScrollTop) {
           console.log("✅ Reached bottom or no new content loaded.");
           break;
       }


       lastScrollTop = container.scrollTop;
       scrollAttempts++;
   }
   console.log(`✅ Finished scrolling. Total attempts: ${scrollAttempts}`);
   return 'done';
}


async function waitForElement(selector, timeout = 30000) {
   return new Promise((resolve, reject) => {
       const startTime = Date.now();
       const interval = setInterval(() => {
           const element = document.querySelector(selector);
           if (element) {
               clearInterval(interval);
               resolve(element);
           } else if (Date.now() - startTime > timeout) {
               clearInterval(interval);
               reject(new Error(`Element with selector "${selector}" not found within ${timeout}ms.`));
           }
       }, 200);
   });
}
function extractCityStateZip(address) {
   // Pattern for full address with city, state, and ZIP
   const fullAddressRegex = /,\s*([^,]+),\s*([A-Z]{2})\s*(\d{5})(?:-\d{4})?$/;


   // Pattern for city, state, and ZIP only
   const cityStateZipRegex = /^([^,]+),\s*([A-Z]{2})\s*(\d{5})(?:-\d{4})?$/;


   // Pattern for state and ZIP only
   const stateZipRegex = /^([A-Z]{2})\s*(\d{5})(?:-\d{4})?$/;


   let match = address.match(fullAddressRegex);
   if (match) {
       return {
           city: match[1].trim(),
           state: match[2],
           zip: match[3]
       };
   }


   match = address.match(cityStateZipRegex);
   if (match) {
       return {
           city: match[1].trim(),
           state: match[2],
           zip: match[3]
       };
   }


   match = address.match(stateZipRegex);
   if (match) {
       return {
           city: null,
           state: match[1],
           zip: match[2]
       };
   }


   return null; // No valid pattern matched
}
function getLatLngFromUrl(url) {
   const match = url.match(/center=([-.\d]+),([-.\d]+)/);
   if (match) {
       return {
           latitude: parseFloat(match[1]),
           longitude: parseFloat(match[2])
       };
   }
   return {latitude: 0,
           longitude: 0}; // Return null if lat/lng not found
}


function extract_fact_and_feature(){
   const result = {};


   document.querySelectorAll('[data-testid="fact-category"]').forEach(category => {


       category.querySelectorAll('li span').forEach(span => {
           const text = span.innerText.trim();
           const parts = text.split(':');
          
           if (parts.length >= 2) {
               let key = parts[0].trim().toLowerCase().replace(/\s+/g, '_').replace(/[^a-z0-9_]/g, '');
               const value = parts.slice(1).join(':').trim();
               result[key] = value;
           } else {
               let key = text.trim().toLowerCase().replace(/\s+/g, '_').replace(/[^a-z0-9_]/g, '');
               result[key] = true;
           }
       });


   });


   console.log(result);
   return result;
}


function extract_tax_history() {
   const table = document.querySelector('table.StyledTableComponents__StyledTable-fshdp-8-111-0__sc-shu7eb-2');
   const results = [];


   if (!table) return results;


   const rows = table.querySelectorAll('tbody tr');


   rows.forEach(row => {
       const cells = row.querySelectorAll('td');
       if (cells.length < 3) return;


       const year = cells[0].innerText.trim();


       const propertyTaxes = cells[1].innerText.trim();


       let taxAssessment = cells[2].innerText.trim();
       const percentSpan = cells[2].querySelector('[data-testid="percent-display"]');
       let percentChange = "";


       if (percentSpan) {
           percentChange = percentSpan.innerText.trim();
           // Remove percentChange from taxAssessment to avoid duplication
           taxAssessment = taxAssessment.replace(percentChange, '').trim();
       }


       results.push({
           year,
           propertyTaxes,
           taxAssessment,
           percentChange
       });
   });


   console.log(results);
   return results;
}
function extractClimateFactors() {
   const factors = [];
   const factorCards = document.querySelectorAll('#climate-hub-grid [data-testid="climate-hub-factor-heading"]');


   factorCards.forEach(card => {
       const nameEl = card.querySelector('[data-testid="climate-hub-factor-name"] span');
       const severityEl = card.querySelector('h4');
       const ratingEl = card.querySelector('span[role="note"] span');


       if (nameEl && severityEl && ratingEl) {
           factors.push({
               factor: nameEl.textContent.trim(),
               severity: severityEl.textContent.trim(),
               rating: ratingEl.textContent.trim()
           });
       }
   });


   return factors;
}




async function startScraping() {
   try {
       const { InfinityscrapingActive } = await chrome.storage.local.get(['InfinityscrapingActive']);
       const { scrapingActive } = await chrome.storage.local.get(['scrapingActive']);


       if (!InfinityscrapingActive && !scrapingActive) {
           return false;
       }


       await scrollToBottom();
       await waitForElement('#grid-search-results .photo-cards.photo-cards_extra-attribution>li');


       const rows = document.querySelectorAll('#grid-search-results .photo-cards.photo-cards_extra-attribution>li');
       if (rows.length === 0) {
           console.log("No listings found after waiting.");
           return null;
       }


       const tableData = [];


       for (const tr of rows) {
            
           const rowData = [];


           const price = tr.querySelector('[data-test="property-card-price"]');
           const property_url = tr.querySelector('[data-test="property-card-link"]');
           const address_elem = tr.querySelector('[data-test="property-card-link"] address');
           const pics = tr.querySelectorAll('picture source');
           const property_by = tr.querySelector('.StyledPropertyCardDataArea-c11n-8-109-3__sc-10i1r6-0.tCgTM');


           const prop_url = property_url ? property_url.href.trim() : '';


           rowData.push({ 'price': price ? price.innerText.trim() : '' });
           rowData.push({ 'property_url': prop_url });
           rowData.push({ 'address': address_elem ? address_elem.innerText.trim() : '' });


           const imageUrls = []; 
           pics.forEach((pic) => { 
               if (pic.srcset) {
                   const urls = pic.srcset.split(',').map(url => url.trim().split(' ')[0]);
                   imageUrls.push(...urls);
               }
           });
           rowData.push({ 'images': imageUrls });
           rowData.push({ 'property_by': property_by ? property_by.innerText.trim() : '' });


           if (ScrapedDetails.includes(prop_url)) {
               console.log("Property already scraped:", prop_url);
               continue;
           }


           ScrapedDetails.push(prop_url);


           if (property_url) {
   property_url.click();
   await new Promise(resolve => setTimeout(resolve, 2000));
} else {
   console.warn("❌ Property link not found, skipping this row.");
   continue;
}
           await new Promise(resolve => setTimeout(resolve, 3000));


           try {
               await waitForElement('[data-testid="content-container"]');
               const newContent = document.querySelector('[data-testid="content-container"]');
               // Scroll the main layout container a little bit to ensure dynamic content loads or buttons are visible
           
               // Fallback: also scroll window a bit
              // window.scrollTo(0, document.body.scrollHeight - 500);
               await new Promise(resolve => setTimeout(resolve, 500));
               if(document.querySelector('[data-testid="description"] button')){
                   document.querySelector('[data-testid="description"] button').click();
                   await new Promise(resolve => setTimeout(resolve, 200));


               }else{
                   console.log("No 'See More Details' button found, skipping.");
               }
                  const layoutContainer = document.querySelector('.layout-container-desktop');
               if (layoutContainer) {
                   layoutContainer.scrollBy({ top: 2200, behavior: 'smooth' });
                   await new Promise(resolve => setTimeout(resolve, 500));
               }
               await new Promise(resolve => setTimeout(resolve, 1000));


               try{
                   if(document.querySelector('[data-testid="facts-and-features-wrapper-footer"] a')){
                   document.querySelector('[data-testid="facts-and-features-wrapper-footer"] a').click();
                   await new Promise(resolve => setTimeout(resolve, 1000));
               }else{
                   console.log("No 'See More Facts and Features' button found, skipping.");
               }
               }catch (error) {
                   console.error("Error clicking 'See More Facts and Features' button:", error);
               }
              




               const description = newContent ? newContent.querySelector('[data-testid="content-container"]') : null;
               const fullAddress=  document.querySelector('[data-testid="fs-chip-container"] h1')? document.querySelector('[data-testid="fs-chip-container"] h1').innerText : '';
               const addressParts = extractCityStateZip(fullAddress);
               const state = addressParts ? addressParts.state : '';
               const zip = addressParts ? addressParts.zip : '';  
               const city = addressParts? addressParts.city : '';
               const bed_bath_sqft = document.querySelectorAll('[data-testid="bed-bath-sqft-fact-container"]') ? document.querySelectorAll('[data-testid="bed-bath-sqft-fact-container"]') : [];
               var bed =''
               var bath = '';
               var sqft = '';
               if (bed_bath_sqft.length > 0) {
                    bed = bed_bath_sqft[0].querySelector('span') ? bed_bath_sqft[0].querySelector('span').innerText : '';
                    bath = bed_bath_sqft[1] ? bed_bath_sqft[1].querySelector('span').innerText : '';
                    sqft = bed_bath_sqft[2] ? bed_bath_sqft[2].querySelector('span').innerText : '';
                  
              
               }
               const row_property_details = document.querySelector('[data-testid="bed-bath-sqft-facts"]')?document.querySelector('[data-testid="bed-bath-sqft-facts"]').innerText.replace(/\n/g, ' '): '';
              
               const price_per_sqft = document.querySelector('[data-testid="price-per-square-foot"]') ? document.querySelector('[data-testid="price-per-square-foot"]').innerText : '';
               const prop_infos = document.querySelectorAll('.sc-iGOTzB.eJeeze .sc-fBDSiZ.jjavTM');
               const type_of_property = prop_infos.length > 0 ? prop_infos[0].innerText : '';
               const year_built = prop_infos.length > 1 ? prop_infos[1].innerText : '';
               const lot_size = prop_infos.length > 2 ? prop_infos[2].innerText : '';
               const zestimate = prop_infos.length > 3 ? prop_infos[3].innerText : '';
               const sqft_price = prop_infos.length > 4 ? prop_infos[4].innerText : '';
               const hoa_fee = prop_infos.length > 5 ? prop_infos[5].innerText : '';


               const date_scraped = new Date().toISOString().split('T')[0]; 
               const map_url = document.querySelector('.media-stream-map.media-stream-visible-map img') ? document.querySelector('.media-stream-map.media-stream-visible-map img').src : '';
              
               const lat = getLatLngFromUrl(map_url).latitude? getLatLngFromUrl(map_url).latitude : '';
               const long = getLatLngFromUrl(map_url).longitude? getLatLngFromUrl(map_url).longitude : '';
               //
               const status = document.querySelector('[data-testid="gallery-status-pill"]') ? document.querySelector('[data-testid="gallery-status-pill"]').innerText : '';
              
               const mortage_payment = document.querySelector('[data-testid="chip-personalize-payment-module"] .Text-c11n-8-111-0__sc-aiai24-0.eIVhby') ? document.querySelector('[data-testid="chip-personalize-payment-module"] .Text-c11n-8-111-0__sc-aiai24-0.eIVhby').innerText : '';
              
               const propert_owner_info = document.querySelector('[data-testid="attribution-PROPERTY_OWNER"]') ? document.querySelector('[data-testid="attribution-PROPERTY_OWNER"]').innerText : '';
               //Estimated Market Value
               const estimated_market_value = document.querySelectorAll('[data-testid="zest-stat-box"]') ? document.querySelectorAll('[data-testid="zest-stat-box"]'): [];
               //Estimated Rent
               const zeEstimated_salesRange = estimated_market_value.length > 0 ? estimated_market_value[1].querySelector('[data-testid="zestimate-range"]') : '';
               const zeEstimated_rent = estimated_market_value.length >1? estimated_market_value[2].querySelector('[data-testid="rent-zestimate"]') : '';
               //FACTS
               const facts = extract_fact_and_feature();


                   const amenities_included = facts.amenities_included || "";
                   const annual_tax_amount = facts.annual_tax_amount || "";
                   const bathrooms = facts.bathrooms || "";
                   const bedrooms = facts.bedrooms || "";
                   const date_on_market = facts.date_on_market || "";
                   const exterior_features = facts.exterior_features || "";
                   const flooring = facts.flooring || "";
                   const full_bathrooms = facts.full_bathrooms || "";
                   const has_garage = facts.has_garage || "";
                   const has_hoa = facts.has_hoa || "";
                   const has_view = facts.has_view || "";
                   const hoa_fee2 = facts.hoa_fee || "";
                   const home_type = facts.home_type || "";
                   const laundry = facts.laundry || "";
                   const parcel_number = facts.parcel_number || "";
                   const parking_features = facts.parking_features || "";
                   const patio__porch = facts.patio__porch || "";
                   const price_per_square_foot = facts.price_per_square_foot || "";
                   const property_subtype = facts.property_subtype || "";
                   const region = facts.region || "";
                   const services_included = facts.services_included || "";
                   const special_conditions = facts.special_conditions || "";
                   const subdivision = facts.subdivision || "";
                   const tax_assessed_value = facts.tax_assessed_value || "";
                   const total_interior_livable_area = facts.total_interior_livable_area || "";
                   const total_structure_area = facts.total_structure_area || "";
                   const view_description = facts.view_description || "";
                   const year_built2 = facts.year_built || "";


                   //Day on Zillow
                const days_in_zillow = document.querySelector('[data-testid="listed-stats-test-id"] dt') ? document.querySelector('.styles__StyledOverviewStats-fshdp-8-111-0__sc-1x11gd9-0.jRtSPq dt').innerText : '';   
               //Tax Histort
               const tax_history = extract_tax_history();
               //clamate
               const climate_factors = extractClimateFactors();
               //
              
               rowData.push({ 'full_address': fullAddress });
               rowData.push({ 'state': state });
               rowData.push({ 'zip': zip });
               rowData.push({ 'city': city });
               rowData.push({ 'beds': bed });
               rowData.push({ 'baths': bath });
               rowData.push({ 'sqft': sqft });
               rowData.push({ 'row_property_details': row_property_details });
               rowData.push({ 'price_per_sqft': price_per_sqft });
               rowData.push({ 'type_of_property': type_of_property });
               rowData.push({ 'year_built': year_built });
               rowData.push({ 'lot_size': lot_size });
               rowData.push({ 'zestimate': zestimate });
               rowData.push({ 'sqft_price': sqft_price });
               rowData.push({ 'hoa_fee': hoa_fee });
               rowData.push({ 'date_scraped': date_scraped });
               rowData.push({ 'map_url': map_url });
               rowData.push({ 'latitude': lat });
               rowData.push({ 'longitude': long });
               rowData.push({ 'status': status });
               rowData.push({ 'mortgage_payment': mortage_payment });
               rowData.push({ 'property_owner_info': propert_owner_info });
               rowData.push({ 'zestimate_sales_range': zeEstimated_salesRange ? zeEstimated_salesRange.innerText.trim() : '' });
               rowData.push({ 'zestimate_rent': zeEstimated_rent ? zeEstimated_rent.innerText.trim() : '' });


               // Pushing data from 'facts'
               rowData.push({ 'amenities_included': amenities_included });
               rowData.push({ 'annual_tax_amount': annual_tax_amount });
               rowData.push({ 'fact_bathrooms': bathrooms });
               rowData.push({ 'fact_bedrooms': bedrooms });
               rowData.push({ 'date_on_market': date_on_market });
               rowData.push({ 'exterior_features': exterior_features });
               rowData.push({ 'flooring': flooring });
               rowData.push({ 'full_bathrooms': full_bathrooms });
               rowData.push({ 'has_garage': has_garage });
               rowData.push({ 'has_hoa': has_hoa });
               rowData.push({ 'has_view': has_view });
               rowData.push({ 'hoa_fee_facts': hoa_fee2 });
               rowData.push({ 'home_type': home_type });
               rowData.push({ 'laundry': laundry });
               rowData.push({ 'parcel_number': parcel_number });
               rowData.push({ 'parking_features': parking_features });
               rowData.push({ 'patio_porch': patio__porch });
               rowData.push({ 'fact_price_per_square_foot': price_per_square_foot });
               rowData.push({ 'property_subtype': property_subtype });
               rowData.push({ 'region': region });
               rowData.push({ 'services_included': services_included });
               rowData.push({ 'special_conditions': special_conditions });
               rowData.push({ 'subdivision': subdivision });
               rowData.push({ 'tax_assessed_value': tax_assessed_value });
               rowData.push({ 'total_interior_livable_area': total_interior_livable_area });
               rowData.push({ 'total_structure_area': total_structure_area });
               rowData.push({ 'view_description': view_description });
               rowData.push({ 'fact_year_built': year_built2 });


               // Pushing other complex data types
               rowData.push({ 'days_in_zillow': days_in_zillow });
               rowData.push({ 'tax_history': tax_history }); // This will be an array of objects
               rowData.push({ 'climate_factors': climate_factors });
               rowData.push({ 'description': description ? description.innerText.trim() : '' });
           } catch (error) {
               // Enhanced error logging for better debugging
               const errorContext = {
                   message: 'Error extracting details',
                   error: error && error.message ? error.message : error,
                   stack: error && error.stack ? error.stack : undefined,
                   time: new Date().toISOString()
               };
               console.error("[Zillow Scraper] Detailed Extraction Error:", errorContext);
               rowData.push({ 'description': '' });
           }


         
           const closeButton = document.querySelector('[data-testid="close-lightbox-button"]');
           if (closeButton) {     
               closeButton.click();


               await new Promise(resolve => setTimeout(resolve, 2000));
           } else{
               const backButton = document.querySelector('[data-testid="desktop-action-bar"] button>span');
               if (backButton) {
               backButton.click();
               await new Promise(resolve => setTimeout(resolve, 2000));
               }
           }




           tableData.push({ rows: rowData });
       }


       console.log("Scraped Data:", tableData);
       return tableData;
   } catch (error) {
       console.error("Error during scraping:", error);
       return null;
   }
}

async function startScrapingAll() {
   try {
    while(true){
       const { InfinityscrapingActive } = await chrome.storage.local.get(['InfinityscrapingActive']);
       const { scrapingActive } = await chrome.storage.local.get(['scrapingActive']);


       if (!InfinityscrapingActive && !scrapingActive) {
           return false;
       }


       await scrollToBottom();
       await waitForElement('#grid-search-results .photo-cards.photo-cards_extra-attribution>li');


       const rows = document.querySelectorAll('#grid-search-results .photo-cards.photo-cards_extra-attribution>li');
       if (rows.length === 0) {
           console.log("No listings found after waiting.");
           return null;
       }


       const tableData = [];


       for (const tr of rows) {
            
           const rowData = [];


           const price = tr.querySelector('[data-test="property-card-price"]');
           const property_url = tr.querySelector('[data-test="property-card-link"]');
           const address_elem = tr.querySelector('[data-test="property-card-link"] address');
           const pics = tr.querySelectorAll('picture source');
           const property_by = tr.querySelector('.StyledPropertyCardDataArea-c11n-8-109-3__sc-10i1r6-0.tCgTM');


           const prop_url = property_url ? property_url.href.trim() : '';


           rowData.push({ 'price': price ? price.innerText.trim() : '' });
           rowData.push({ 'property_url': prop_url });
           rowData.push({ 'address': address_elem ? address_elem.innerText.trim() : '' });


           const imageUrls = []; 
           pics.forEach((pic) => { 
               if (pic.srcset) {
                   const urls = pic.srcset.split(',').map(url => url.trim().split(' ')[0]);
                   imageUrls.push(...urls);
               }
           });
           rowData.push({ 'images': imageUrls });
           rowData.push({ 'property_by': property_by ? property_by.innerText.trim() : '' });


           if (ScrapedDetails.includes(prop_url)) {
               console.log("Property already scraped:", prop_url);
               continue;
           }


           ScrapedDetails.push(prop_url);


           if (property_url) {
   property_url.click();
   await new Promise(resolve => setTimeout(resolve, 2000));
} else {
   console.warn("❌ Property link not found, skipping this row.");
   continue;
}
           await new Promise(resolve => setTimeout(resolve, 3000));


           try {
               await waitForElement('[data-testid="content-container"]');
               const newContent = document.querySelector('[data-testid="content-container"]');
               // Scroll the main layout container a little bit to ensure dynamic content loads or buttons are visible
           
               // Fallback: also scroll window a bit
              // window.scrollTo(0, document.body.scrollHeight - 500);
               await new Promise(resolve => setTimeout(resolve, 500));
               if(document.querySelector('[data-testid="description"] button')){
                   document.querySelector('[data-testid="description"] button').click();
                   await new Promise(resolve => setTimeout(resolve, 200));


               }else{
                   console.log("No 'See More Details' button found, skipping.");
               }
                  const layoutContainer = document.querySelector('.layout-container-desktop');
               if (layoutContainer) {
                   layoutContainer.scrollBy({ top: 2200, behavior: 'smooth' });
                   await new Promise(resolve => setTimeout(resolve, 500));
               }
               await new Promise(resolve => setTimeout(resolve, 1000));


               try{
                   if(document.querySelector('[data-testid="facts-and-features-wrapper-footer"] a')){
                   document.querySelector('[data-testid="facts-and-features-wrapper-footer"] a').click();
                   await new Promise(resolve => setTimeout(resolve, 1000));
               }else{
                   console.log("No 'See More Facts and Features' button found, skipping.");
               }
               }catch (error) {
                   console.error("Error clicking 'See More Facts and Features' button:", error);
               }
              




               const description = newContent ? newContent.querySelector('[data-testid="content-container"]') : null;
               const fullAddress=  document.querySelector('[data-testid="fs-chip-container"] h1')? document.querySelector('[data-testid="fs-chip-container"] h1').innerText : '';
               const addressParts = extractCityStateZip(fullAddress);
               const state = addressParts ? addressParts.state : '';
               const zip = addressParts ? addressParts.zip : '';  
               const city = addressParts? addressParts.city : '';
               const bed_bath_sqft = document.querySelectorAll('[data-testid="bed-bath-sqft-fact-container"]') ? document.querySelectorAll('[data-testid="bed-bath-sqft-fact-container"]') : [];
               var bed =''
               var bath = '';
               var sqft = '';
               if (bed_bath_sqft.length > 0) {
                    bed = bed_bath_sqft[0].querySelector('span') ? bed_bath_sqft[0].querySelector('span').innerText : '';
                    bath = bed_bath_sqft[1] ? bed_bath_sqft[1].querySelector('span').innerText : '';
                    sqft = bed_bath_sqft[2] ? bed_bath_sqft[2].querySelector('span').innerText : '';
                  
              
               }
               const row_property_details = document.querySelector('[data-testid="bed-bath-sqft-facts"]')?document.querySelector('[data-testid="bed-bath-sqft-facts"]').innerText.replace(/\n/g, ' '): '';
              
               const price_per_sqft = document.querySelector('[data-testid="price-per-square-foot"]') ? document.querySelector('[data-testid="price-per-square-foot"]').innerText : '';
               const prop_infos = document.querySelectorAll('.sc-iGOTzB.eJeeze .sc-fBDSiZ.jjavTM');
               const type_of_property = prop_infos.length > 0 ? prop_infos[0].innerText : '';
               const year_built = prop_infos.length > 1 ? prop_infos[1].innerText : '';
               const lot_size = prop_infos.length > 2 ? prop_infos[2].innerText : '';
               const zestimate = prop_infos.length > 3 ? prop_infos[3].innerText : '';
               const sqft_price = prop_infos.length > 4 ? prop_infos[4].innerText : '';
               const hoa_fee = prop_infos.length > 5 ? prop_infos[5].innerText : '';


               const date_scraped = new Date().toISOString().split('T')[0]; 
               const map_url = document.querySelector('.media-stream-map.media-stream-visible-map img') ? document.querySelector('.media-stream-map.media-stream-visible-map img').src : '';
              
               const lat = getLatLngFromUrl(map_url).latitude? getLatLngFromUrl(map_url).latitude : '';
               const long = getLatLngFromUrl(map_url).longitude? getLatLngFromUrl(map_url).longitude : '';
               //
               const status = document.querySelector('[data-testid="gallery-status-pill"]') ? document.querySelector('[data-testid="gallery-status-pill"]').innerText : '';
              
               const mortage_payment = document.querySelector('[data-testid="chip-personalize-payment-module"] .Text-c11n-8-111-0__sc-aiai24-0.eIVhby') ? document.querySelector('[data-testid="chip-personalize-payment-module"] .Text-c11n-8-111-0__sc-aiai24-0.eIVhby').innerText : '';
              
               const propert_owner_info = document.querySelector('[data-testid="attribution-PROPERTY_OWNER"]') ? document.querySelector('[data-testid="attribution-PROPERTY_OWNER"]').innerText : '';
               //Estimated Market Value
               const estimated_market_value = document.querySelectorAll('[data-testid="zest-stat-box"]') ? document.querySelectorAll('[data-testid="zest-stat-box"]'): [];
               //Estimated Rent
               const zeEstimated_salesRange = estimated_market_value.length > 0 ? estimated_market_value[1].querySelector('[data-testid="zestimate-range"]') : '';
               const zeEstimated_rent = estimated_market_value.length >1? estimated_market_value[2].querySelector('[data-testid="rent-zestimate"]') : '';
               //FACTS
               const facts = extract_fact_and_feature();


                   const amenities_included = facts.amenities_included || "";
                   const annual_tax_amount = facts.annual_tax_amount || "";
                   const bathrooms = facts.bathrooms || "";
                   const bedrooms = facts.bedrooms || "";
                   const date_on_market = facts.date_on_market || "";
                   const exterior_features = facts.exterior_features || "";
                   const flooring = facts.flooring || "";
                   const full_bathrooms = facts.full_bathrooms || "";
                   const has_garage = facts.has_garage || "";
                   const has_hoa = facts.has_hoa || "";
                   const has_view = facts.has_view || "";
                   const hoa_fee2 = facts.hoa_fee || "";
                   const home_type = facts.home_type || "";
                   const laundry = facts.laundry || "";
                   const parcel_number = facts.parcel_number || "";
                   const parking_features = facts.parking_features || "";
                   const patio__porch = facts.patio__porch || "";
                   const price_per_square_foot = facts.price_per_square_foot || "";
                   const property_subtype = facts.property_subtype || "";
                   const region = facts.region || "";
                   const services_included = facts.services_included || "";
                   const special_conditions = facts.special_conditions || "";
                   const subdivision = facts.subdivision || "";
                   const tax_assessed_value = facts.tax_assessed_value || "";
                   const total_interior_livable_area = facts.total_interior_livable_area || "";
                   const total_structure_area = facts.total_structure_area || "";
                   const view_description = facts.view_description || "";
                   const year_built2 = facts.year_built || "";


                   //Day on Zillow
                const days_in_zillow = document.querySelector('[data-testid="listed-stats-test-id"] dt') ? document.querySelector('.styles__StyledOverviewStats-fshdp-8-111-0__sc-1x11gd9-0.jRtSPq dt').innerText : '';   
               //Tax Histort
               const tax_history = extract_tax_history();
               //clamate
               const climate_factors = extractClimateFactors();
               //
              
               rowData.push({ 'full_address': fullAddress });
               rowData.push({ 'state': state });
               rowData.push({ 'zip': zip });
               rowData.push({ 'city': city });
               rowData.push({ 'beds': bed });
               rowData.push({ 'baths': bath });
               rowData.push({ 'sqft': sqft });
               rowData.push({ 'row_property_details': row_property_details });
               rowData.push({ 'price_per_sqft': price_per_sqft });
               rowData.push({ 'type_of_property': type_of_property });
               rowData.push({ 'year_built': year_built });
               rowData.push({ 'lot_size': lot_size });
               rowData.push({ 'zestimate': zestimate });
               rowData.push({ 'sqft_price': sqft_price });
               rowData.push({ 'hoa_fee': hoa_fee });
               rowData.push({ 'date_scraped': date_scraped });
               rowData.push({ 'map_url': map_url });
               rowData.push({ 'latitude': lat });
               rowData.push({ 'longitude': long });
               rowData.push({ 'status': status });
               rowData.push({ 'mortgage_payment': mortage_payment });
               rowData.push({ 'property_owner_info': propert_owner_info });
               rowData.push({ 'zestimate_sales_range': zeEstimated_salesRange ? zeEstimated_salesRange.innerText.trim() : '' });
               rowData.push({ 'zestimate_rent': zeEstimated_rent ? zeEstimated_rent.innerText.trim() : '' });


               // Pushing data from 'facts'
               rowData.push({ 'amenities_included': amenities_included });
               rowData.push({ 'annual_tax_amount': annual_tax_amount });
               rowData.push({ 'fact_bathrooms': bathrooms });
               rowData.push({ 'fact_bedrooms': bedrooms });
               rowData.push({ 'date_on_market': date_on_market });
               rowData.push({ 'exterior_features': exterior_features });
               rowData.push({ 'flooring': flooring });
               rowData.push({ 'full_bathrooms': full_bathrooms });
               rowData.push({ 'has_garage': has_garage });
               rowData.push({ 'has_hoa': has_hoa });
               rowData.push({ 'has_view': has_view });
               rowData.push({ 'hoa_fee_facts': hoa_fee2 });
               rowData.push({ 'home_type': home_type });
               rowData.push({ 'laundry': laundry });
               rowData.push({ 'parcel_number': parcel_number });
               rowData.push({ 'parking_features': parking_features });
               rowData.push({ 'patio_porch': patio__porch });
               rowData.push({ 'fact_price_per_square_foot': price_per_square_foot });
               rowData.push({ 'property_subtype': property_subtype });
               rowData.push({ 'region': region });
               rowData.push({ 'services_included': services_included });
               rowData.push({ 'special_conditions': special_conditions });
               rowData.push({ 'subdivision': subdivision });
               rowData.push({ 'tax_assessed_value': tax_assessed_value });
               rowData.push({ 'total_interior_livable_area': total_interior_livable_area });
               rowData.push({ 'total_structure_area': total_structure_area });
               rowData.push({ 'view_description': view_description });
               rowData.push({ 'fact_year_built': year_built2 });


               // Pushing other complex data types
               rowData.push({ 'days_in_zillow': days_in_zillow });
               rowData.push({ 'tax_history': tax_history }); // This will be an array of objects
               rowData.push({ 'climate_factors': climate_factors });
               rowData.push({ 'description': description ? description.innerText.trim() : '' });
           } catch (error) {
               // Enhanced error logging for better debugging
               const errorContext = {
                   message: 'Error extracting details',
                   error: error && error.message ? error.message : error,
                   stack: error && error.stack ? error.stack : undefined,
                   time: new Date().toISOString()
               };
               console.error("[Zillow Scraper] Detailed Extraction Error:", errorContext);
               rowData.push({ 'description': '' });
           }


         
           const closeButton = document.querySelector('[data-testid="close-lightbox-button"]');
           if (closeButton) {     
               closeButton.click();


               await new Promise(resolve => setTimeout(resolve, 2000));
           } else{
               const backButton = document.querySelector('[data-testid="desktop-action-bar"] button>span');
               if (backButton) {
               backButton.click();
               await new Promise(resolve => setTimeout(resolve, 2000));
               }
           }




           tableData.push({ rows: rowData });
       }


       console.log("Scraped Data:", tableData);
       try{
         await document.waitForElement('[rel="next"]');
         await document.querySelector('[rel="next"]').click;
       }catch(e){
        break;
       }
       await new Promise(resolve => setTimeout(resolve, 20000));
   
   } } catch (error) {
       console.error("Error during scraping:", error);
       return null;
   }
}

function flattenDataToSingleArray(data) {
   if (!data || data.length === 0) return [];
   const flattenedArray = [];
   data.forEach((row, rowIndex) => {
       if (Array.isArray(row)) {
           const tempRowArray = [];
           row.forEach(cell => {
               if (Array.isArray(cell)) {
                   cell.forEach(value => tempRowArray.push(value));
               } else if (cell !== null && cell !== undefined) {
                   tempRowArray.push(cell);
               }
           });
           if (rowIndex === 2 && tempRowArray.length === 2) {
               tempRowArray.push("");
           }
           tempRowArray.forEach(value => flattenedArray.push(value));
       }
   });
   return flattenedArray;
}


function sendDataToPopup(data) {
   accumulatedData = data;
   chrome.storage.local.set({ accumulatedData }, () => {});


   const pages_scraped = parseInt(localStorage.getItem('pages_scraped') || '1', 10);
   const rows_from_last_page = parseInt(localStorage.getItem('rows_from_last_page') || '1', 10);
   localStorage.setItem('pages_scraped', pages_scraped + 1);
   localStorage.setItem('rows_collected', 1);
   localStorage.setItem('rows_from_last_page', 12);


   chrome.runtime.sendMessage({
       action: "updatePopupx",
       payload: data,
       pages_scraped: (pages_scraped + 1).toString(),
       rows_collected: '12',
       rows_from_last_page: '14'
   });
}


async function initiateScraping() {
   const scrapedData = await startScraping();
   if (scrapedData === false || !scrapedData) return;


   const ddd = scrapedData.map(rowObj => flattenDataToSingleArray(rowObj.rows));
   sendDataToPopup(scrapedData);
}


chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
   if (request.action === 'resetAccumulatedData') {
       accumulatedData = [];
       chrome.storage.local.set({ accumulatedData: [] });
   } else if (request.action === 'resetStat') {
       localStorage.setItem('pages_scraped', 2);
       localStorage.setItem('rows_collected', 0);
       localStorage.setItem('rows_from_last_page', 25);
   } else if (request.action === "run_scraper") {
       if (!window.hasAlreadyScraped) {
           window.hasAlreadyScraped = true;
           const scrapedData = await startScraping();
           sendDataToPopup(scrapedData);
       }
   }
});


chrome.storage.local.get(['accumulatedData'], (result) => {
   if (result.accumulatedData) {
       accumulatedData = result.accumulatedData;
   }
});


if (window.location.href.includes('/homes/') && !window.hasAlreadyScraped) {
   window.hasAlreadyScraped = true;
   initiateScraping();
}



