add KML generation options

Changelog.md

@@ -1,6 +1,11 @@
 
 # CHANGELOGS
 
+## [0.2.5] - 2024-09-30
+
+- Add option to download KML file For 2G, 3G, and LTE
+- Fix bug when LTE name is empty
+
 ## [0.2.3] - 2024-09-30
 
 - Moving physical database to github storage repository

README.md

@@ -44,6 +44,7 @@ You can access the hosted version of the app at [https://davmelchi-db-query.hf.s
 - [x] Add  MRBTS with code
 - [x] Check TCH from MAL sheet
 - [x] Add Analitic dashboards for each database (Count of NE)
+- [x] Add kml generation
 - [ ] Improve Dashboard
 - [ ] Add the ability to select columns
 - [ ] Error handling

app.py

@@ -6,7 +6,7 @@ st.set_page_config(
     layout="wide",
     initial_sidebar_state="expanded",
     menu_items={
-        "About": "**📡 NPO DB Query v0.2.4**",
+        "About": "**📡 NPO DB Query v0.2.5**",
     },
 )
 

apps/database_page.py

@@ -6,16 +6,16 @@ from st_aggrid import AgGrid, ColumnsAutoSizeMode
 
 from apps.dump_analysis import dump_analysis_space
 from queries.process_all_db import process_all_tech_db, process_all_tech_db_with_stats
-from queries.process_gsm import process_gsm_data_to_excel
+from queries.process_gsm import process_gsm_data_to_excel, process_gsm_data_to_kml
 from queries.process_invunit import process_invunit_data_to_excel
-from queries.process_lte import process_lte_data_to_excel
+from queries.process_lte import process_lte_data_to_excel, process_lte_data_to_kml
 
 # from queries.process_mal import process_mal_data_to_excel
 from queries.process_mrbts import process_mrbts_data_to_excel
 from queries.process_neighbors import process_neighbors_data_to_excel
 
 # from queries.process_trx import process_trx_with_bts_name_data_to_excel
-from queries.process_wcdma import process_wcdma_data_to_excel
+from queries.process_wcdma import process_wcdma_data_to_excel, process_wcdma_data_to_kml
 from utils.check_sheet_exist import DumpType, Technology, execute_checks_sheets_exist
 from utils.utils_vars import GsmAnalysisData, UtilsVars, WcdmaAnalysisData
 
@@ -67,6 +67,37 @@ def download_button(database_type):
     )
 
 
+def process_kml_database(process_func, database_type):
+    if uploaded_file is not None:
+        start_time = time.time()
+        process_func(uploaded_file)
+        execution_time = time.time() - start_time
+        st.write(
+            f"{database_type} database is generated. Execution time: {execution_time:.2f} seconds"
+        )
+        kml_download_button(database_type)
+
+
+@st.fragment()
+def kml_download_button(database_type):
+    if database_type == "2G":
+        data = UtilsVars.gsm_kml_file
+        file_name = f"2G kml_{datetime.now()}.kml"
+    elif database_type == "3G":
+        data = UtilsVars.wcdma_kml_file
+        file_name = f"3G kml_{datetime.now()}.kml"
+    elif database_type == "LTE":
+        data = UtilsVars.lte_kml_file
+        file_name = f"LTE kml_{datetime.now()}.kml"
+    st.download_button(
+        type="primary",
+        label=f"Download {database_type} KML File",
+        data=data,
+        file_name=file_name,
+        mime="application/vnd.google-earth.kml+xml",
+    )
+
+
 def execute_process_all_tech_db(uploaded_file):
     if uploaded_file is not None:
         start_time = time.time()
@@ -94,8 +125,8 @@ def execute_process_all_tech_db_with_stats(uploaded_file: str):
         download_button("All")
 
 
-col1, col2, col3, col4 = st.columns(4)
-col5, col6, col7, col8 = st.columns(4)
+col1, col2, col3, col4, col5 = st.columns(5)
+col6, col7, col8, col9, col10 = st.columns(5)
 if uploaded_file is not None:
     # UtilsVars.file_path = uploaded_file
 
@@ -183,6 +214,32 @@ if uploaded_file is not None:
                         process_neighbors_data_to_excel, "NEI"
                     ),
                 )
+        if Technology.gsm == True:
+            with col8:
+                st.button(
+                    "Generate 2G KML",
+                    on_click=lambda: process_kml_database(
+                        process_gsm_data_to_kml, "2G"
+                    ),
+                )
+
+        if Technology.wcdma == True:
+            with col9:
+                st.button(
+                    "Generate 3G KML",
+                    on_click=lambda: process_kml_database(
+                        process_wcdma_data_to_kml, "3G"
+                    ),
+                )
+
+        if Technology.lte == True:
+            with col10:
+                st.button(
+                    "Generate LTE KML",
+                    on_click=lambda: process_kml_database(
+                        process_lte_data_to_kml, "LTE"
+                    ),
+                )
 
     except Exception as e:
         st.error(f"Error: {e}")

queries/process_gsm.py

@@ -4,6 +4,7 @@ from queries.process_mal import process_mal_data, process_mal_with_bts_name
 from queries.process_trx import process_trx_data, process_trx_with_bts_name
 from utils.config_band import config_band
 from utils.convert_to_excel import convert_dfs, save_dataframe
+from utils.kml_creator import generate_kml_from_df
 from utils.utils_vars import GsmAnalysisData, UtilsVars, get_physical_db
 
 BTS_COLUMNS = [
@@ -56,6 +57,23 @@ BCF_COLUMNS = [
     "site_name",
 ]
 
+GSM_KML_COLUMNS = [
+    "code",
+    "name",
+    "Longitude",
+    "Latitude",
+    "Azimut",
+    "Hauteur",
+    "BSIC",
+    "cellId",
+    "locationAreaIdLAC",
+    "band",
+    "BCCH",
+    "TRX_TCH",
+    "number_trx_per_cell",
+    "number_trx_per_site",
+]
+
 
 def compare_trx_tch_versus_mal(tch1, tch2):
     # Split the strings by commas, convert to sets, and compare
@@ -185,6 +203,22 @@ def process_gsm_data_to_excel(file_path: str):
     UtilsVars.final_gsm_database = convert_dfs(gsm_dfs, ["GSM", "MAL", "TRX"])
 
 
+############################# KML CREATION #################################
+
+
+def process_gsm_data_to_kml(file_path: str):
+    gsm_kml_df = process_gsm_data(file_path)
+    gsm_kml_df = gsm_kml_df[GSM_KML_COLUMNS]
+    # Add colors column base on "band" column
+    gsm_kml_df["color"] = gsm_kml_df["band"].map(UtilsVars.color_mapping)
+    # Add size column base on "band" column
+    gsm_kml_df["size"] = gsm_kml_df["band"].map(UtilsVars.size_mapping)
+    # Remove empty rows
+    gsm_kml_df = gsm_kml_df.dropna(subset=["Longitude", "Latitude", "Azimut"])
+    # Generate kml
+    UtilsVars.gsm_kml_file = generate_kml_from_df(gsm_kml_df)
+
+
 #############################GSM ANALYSIS#################################
 
 

queries/process_invunit.py

@@ -76,6 +76,11 @@ def process_invunit_data(file_path: str) -> pd.DataFrame:
     df_invunit = create_invunit_summary(df_invunit)
     df_invunit["CODE"] = df_invunit["MRBTS"].apply(extract_code_from_mrbts)
     df_invunit = df_invunit[["MRBTS", "CODE", "invunit_summary"]]
+    # df_invunit = df_invunit.sort_values(
+    #     by=["MRBTS", "CODE"],
+    # )
+    # df_invunit["MRBTS_NUM"] = df_invunit.groupby("CODE").cumcount() + 1
+    # df_invunit["MRBTS_NUM"] = "MRBTS_" + df_invunit["MRBTS_NUM"].astype(str)
 
     UtilsVars.all_db_dfs.append(df_invunit)
     UtilsVars.all_db_dfs_names.append("INVUNIT")

queries/process_lte.py

@@ -3,6 +3,7 @@ import pandas as pd
 
 from utils.config_band import config_band
 from utils.convert_to_excel import convert_dfs, save_dataframe
+from utils.kml_creator import generate_kml_from_df
 from utils.utils_vars import (
     LteFddAnalysisData,
     LteTddAnalysisData,
@@ -93,6 +94,21 @@ LNCEL_TDD_COLUMNS = [
     "rootSeqIndex",
 ]
 
+LTE_KML_COLUMNS = [
+    "code",
+    "final_name",
+    "Longitude",
+    "Latitude",
+    "Azimut",
+    "Hauteur",
+    "lcrId",
+    "pMax",
+    "phyCellId",
+    "tac",
+    "rootSeqIndex",
+    "band",
+]
+
 
 def process_lncel(file_path: str):
     """
@@ -113,6 +129,8 @@ def process_lncel(file_path: str):
     df_lncel = dfs["LNCEL"]
     df_lncel.columns = df_lncel.columns.str.replace(r"[ ]", "", regex=True)
     df_lncel["final_name"] = df_lncel["name"].fillna(df_lncel["cellName"])
+    # Replace empty by 999999_empty
+    # df_lncel["final_name"] = df_lncel["final_name"].fillna("999999_empty")
     df_lncel["code"] = df_lncel["final_name"].str.split("_").str[0]
     df_lncel["code"] = (
         pd.to_numeric(df_lncel["code"], errors="coerce").fillna(0).astype(int)
@@ -220,6 +238,31 @@ def process_lte_data_to_excel(file_path: str):
     UtilsVars.final_lte_database = convert_dfs(lte_dfs, ["LTE_FDD", "LTE_TDD"])
 
 
+############################# KML CREATION #################################
+def process_lte_data_to_kml(file_path: str):
+    lte_kml_dfs = process_lte_data(file_path)
+
+    lte_fdd_klm_df = lte_kml_dfs[0]
+    lte_fdd_klm_df = lte_fdd_klm_df[LTE_KML_COLUMNS]
+
+    lte_tdd_klm_df = lte_kml_dfs[1]
+    lte_tdd_klm_df = lte_tdd_klm_df[LTE_KML_COLUMNS]
+
+    # Merge FDD and TDD dataframes
+    lte_kml_df = pd.concat([lte_fdd_klm_df, lte_tdd_klm_df], ignore_index=True)
+
+    # Rename "final_name" to "name"
+    lte_kml_df.rename(columns={"final_name": "name"}, inplace=True)
+    # Add colors column base on "band" column
+    lte_kml_df["color"] = lte_kml_df["band"].map(UtilsVars.color_mapping)
+    # Add size column base on "band" column
+    lte_kml_df["size"] = lte_kml_df["band"].map(UtilsVars.size_mapping)
+    # Remove empty rows
+    lte_kml_df = lte_kml_df.dropna(subset=["Longitude", "Latitude", "Azimut"])
+    # Generate kml
+    UtilsVars.lte_kml_file = generate_kml_from_df(lte_kml_df)
+
+
 #############################LTE ANALYSIS#################################
 
 

queries/process_wcdma.py

@@ -3,6 +3,7 @@ import pandas as pd
 from utils.config_band import config_band
 from utils.convert_to_excel import convert_dfs, save_dataframe
 from utils.extract_code import extract_code_from_mrbts
+from utils.kml_creator import generate_kml_from_df
 from utils.utils_vars import UtilsVars, WcdmaAnalysisData, get_physical_db
 
 WCEL_COLUMNS = [
@@ -76,6 +77,21 @@ WNCEL_COLUMNS = [
     "maxCarrierPower",
 ]
 
+WCDMA_KML_COLUMNS = [
+    "code",
+    "name",
+    "Longitude",
+    "Latitude",
+    "Azimut",
+    "Hauteur",
+    "LAC",
+    "CId",
+    "LAC",
+    "UARFCN",
+    "PriScrCode",
+    "band",
+]
+
 
 def process_wcdma_data(file_path: str):
     """
@@ -183,6 +199,28 @@ def process_wcdma_data_to_excel(file_path: str):
     UtilsVars.final_wcdma_database = convert_dfs([wcdma_dfs], ["WCDMA"])
 
 
+############################# KML CREATION #################################
+
+
+def process_wcdma_data_to_kml(file_path: str):
+    """
+    Process WCDMA data from the specified file path and convert it to KML format
+
+    Args:
+        file_path (str): The path to the file.
+    """
+    wcdma_kml_df = process_wcdma_data(file_path)
+    wcdma_kml_df = wcdma_kml_df[WCDMA_KML_COLUMNS]
+    # Add colors column base on "band" column
+    wcdma_kml_df["color"] = wcdma_kml_df["band"].map(UtilsVars.color_mapping)
+    # Add size column base on "band" column
+    wcdma_kml_df["size"] = wcdma_kml_df["band"].map(UtilsVars.size_mapping)
+    # Remove empty rows
+    wcdma_kml_df = wcdma_kml_df.dropna(subset=["Longitude", "Latitude", "Azimut"])
+    # Generate kml
+    UtilsVars.wcdma_kml_file = generate_kml_from_df(wcdma_kml_df)
+
+
 ############################ANALYTICSS AND STATISTICS############################
 
 

test.py

@@ -19,7 +19,7 @@ class TestProcessAllDB:
     def test_all_dbs(self):
         filepath = r"C:\Users\David\Documents\PROJECTS\2023\PROJET 2023\DUMP\DUMP\NOVEMBRE\20241127_21145_27112024_Dump.xml.gz.xlsb"
         all_dbs(filepath)
-        assert len(UtilsVars.all_db_dfs) == 7
+        assert len(UtilsVars.all_db_dfs) == 8
         assert isinstance(UtilsVars.all_db_dfs[0], pd.DataFrame)
 
     def test_process_all_tech_db(self):

utils/kml_creator.py

@@ -0,0 +1,79 @@
+import io
+import math
+
+import numpy as np
+import pandas as pd
+import simplekml
+
+
+def create_sector(kml: simplekml.Kml, row, arc_angle=65):
+    """Create a sector shape for the telecom antenna in KML with sector details."""
+    code, name, azimuth, lon, lat, size, color = (
+        row["code"],
+        row["name"],
+        row["Azimut"],
+        row["Longitude"],
+        row["Latitude"],
+        row["size"],
+        row["color"],
+    )
+
+    num_points = 20  # Number of points for smooth arc
+    start_angle = azimuth - (arc_angle / 2)
+    end_angle = azimuth + (arc_angle / 2)
+
+    coords = [(lon, lat)]  # Start with the site location (center point)
+
+    # Generate points for the sector arc
+    for angle in np.linspace(start_angle, end_angle, num_points):
+        angle_rad = math.radians(angle)
+        arc_lon = lon + (size / 111320) * math.sin(angle_rad)
+        arc_lat = lat + (size / 111320) * math.cos(angle_rad)
+        coords.append((arc_lon, arc_lat))
+
+    coords.append((lon, lat))  # Close the polygon
+
+    # Create the sector polygon
+    pol = kml.newpolygon(name=name, outerboundaryis=coords)
+
+    # Dynamically create the description from all DataFrame columns
+    description = "<b>Sector Details:</b><br>"
+    for column, value in row.items():
+        description += f"<b>{column}:</b> {value}<br>"
+
+    pol.description = description
+    pol.style.polystyle.color = color  # Set color from DataFrame
+    pol.style.polystyle.outline = 1  # Outline enabled
+    pol.style.linestyle.color = "ff000000"  # Black outline
+
+
+def generate_kml_from_df(df: pd.DataFrame):
+    """Generate a KML file from a Pandas DataFrame for telecom sectors."""
+    kml = simplekml.Kml()
+    site_added = set()  # Keep track of sites already added to avoid duplicates
+
+    # Sort the DataFrame to ensure 900 MHz (smaller) is drawn last (on top)
+    df_sorted = df.sort_values(
+        by="size", ascending=False
+    )  # Larger first, smaller on top
+
+    for _, row in df_sorted.iterrows():
+        code, lon, lat = row["code"], row["Longitude"], row["Latitude"]
+
+        # Add site name as a point only once
+        if code not in site_added:
+            pnt = kml.newpoint(name=code, coords=[(lon, lat)])
+            pnt.style.iconstyle.icon.href = (
+                "http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png"
+            )
+            pnt.style.labelstyle.scale = 1.2  # Adjust label size
+            pnt.description = f"Site: {code}<br>Location: {lat}, {lon}"
+            site_added.add(code)
+
+        create_sector(kml, row)
+
+    kml_data = io.BytesIO()
+    kml_str = kml.kml()  # Get KML as string
+    kml_data.write(kml_str.encode("utf-8"))  # Write KML to BytesIO
+    kml_data.seek(0)  # Move to beginning of BytesIO
+    return kml_data

utils/utils_vars.py

@@ -33,6 +33,24 @@ class UtilsVars:
         10787: "OML UTRA Band I",
         10837: "OML UTRA Band I",
     }
+    color_mapping = {
+        "U900": "7fff0000",
+        "U2100": "7f00ff00",
+        "G900": "7fff0000",
+        "G1800": "7f00ff00",
+        "L800": "7fff0000",
+        "L1800": "7f00ff00",
+        "L2300": "7f00ffff",
+    }
+    size_mapping = {
+        "U900": 100,
+        "U2100": 120,
+        "G900": 100,
+        "G1800": 120,
+        "L800": 100,
+        "L1800": 120,
+        "L2300": 90,
+    }
     wcdma_band = {
         3004: "U900",
         3006: "U900",
@@ -65,6 +83,9 @@ class UtilsVars:
     final_all_database = None
     neighbors_database = ""
     file_path = ""
+    gsm_kml_file = None
+    wcdma_kml_file = None
+    lte_kml_file = None
     # physisal_db = get_physical_db()