silas.brito/netbackup-insights
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: initial architecture setup with Docker, SQLite persistence and PDF tracking

Browse Source
This commit is contained in:
Silas Brito
2026-07-13 18:58:05 -03:00
commit 27652716d7
10 changed files with 1365 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.py
+547
View File
@@ -0,0 +1,547 @@
import streamlit as st
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from parser import parse_nbu_csv, compute_hash
import database as db
import report_gen as rg
import time
import io
import os
# Initialize database
db.init_db()
# Set up page configurations
st.set_page_config(
page_title="NetBackup Log Insights",
page_icon="⚡",
layout="wide",
initial_sidebar_state="expanded"
)
# Custom CSS for high-fidelity dark corporate theme (CXP-inspired)
custom_css = """
<style>
/* Main App Background & Text */
.stApp {
background-color: #0B0F19;
color: #E2E8F0;
font-family: 'Inter', -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
}
/* Headers styling */
h1, h2, h3, h4, h5, h6 {
color: #FFFFFF !important;
font-weight: 700 !important;
}
/* Sidebar Styling */
section[data-testid="stSidebar"] {
background-color: #0F172A !important;
border-right: 1px solid #1E293B;
}
section[data-testid="stSidebar"] h1,
section[data-testid="stSidebar"] h2,
section[data-testid="stSidebar"] h3 {
color: #00D2FF !important;
}
/* Metric Card Styling */
div[data-testid="stMetric"] {
background-color: #1E2640 !important;
border: 1px solid #2E3A5F;
border-radius: 12px;
padding: 20px !important;
box-shadow: 0 8px 16px rgba(0, 0, 0, 0.3);
transition: all 0.3s ease;
}
div[data-testid="stMetric"]:hover {
border-color: #00D2FF;
transform: translateY(-2px);
}
div[data-testid="stMetric"] label {
color: #94A3B8 !important;
font-size: 0.85rem !important;
text-transform: uppercase;
letter-spacing: 0.08em;
font-weight: 600;
}
div[data-testid="stMetric"] div[data-testid="stMetricValue"] {
color: #FFFFFF !important;
font-size: 2.2rem !important;
font-weight: 800;
}
/* Form inputs and buttons styling */
.stSelectbox, .stTextInput, .stTextArea, .stFileUploader {
background-color: #1E2640 !important;
color: #FFFFFF !important;
border-radius: 8px;
}
/* Buttons with neon cyan gradient */
div.stButton > button, div.stDownloadButton > button {
background: linear-gradient(135deg, #0052CC 0%, #00D2FF 100%) !important;
color: #FFFFFF !important;
border: none !important;
border-radius: 8px !important;
padding: 0.6rem 1.8rem !important;
font-weight: 700 !important;
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important;
box-shadow: 0 4px 14px rgba(0, 210, 255, 0.25) !important;
text-transform: uppercase;
font-size: 0.85rem;
letter-spacing: 0.05em;
}
div.stButton > button:hover, div.stDownloadButton > button:hover {
background: linear-gradient(135deg, #00D2FF 0%, #0052CC 100%) !important;
transform: translateY(-2px);
box-shadow: 0 6px 22px rgba(0, 210, 255, 0.5) !important;
}
/* Tabs selector customization */
button[data-baseweb="tab"] {
color: #94A3B8 !important;
font-size: 1.1rem !important;
font-weight: 600 !important;
padding: 10px 20px !important;
border-bottom: 3px solid transparent !important;
transition: all 0.2s ease !important;
}
button[data-baseweb="tab"][aria-selected="true"] {
color: #00D2FF !important;
border-bottom: 3px solid #00D2FF !important;
background-color: rgba(30, 38, 64, 0.2) !important;
}
/* Table headers customize */
div[data-testid="stDataFrame"] {
background-color: #1E2640 !important;
border: 1px solid #2E3A5F;
border-radius: 12px;
padding: 8px;
}
</style>
"""
st.markdown(custom_css, unsafe_allow_html=True)
# Sidebar Design
st.sidebar.markdown("<h1 style='text-align: center; margin-bottom: 10px;'>NBU Insights</h1>", unsafe_allow_html=True)
st.sidebar.markdown("<p style='text-align: center; color: #94A3B8; font-size: 0.9rem; margin-bottom: 30px;'>Docker Cloud Dashboard</p>", unsafe_allow_html=True)
# Global view selector
st.sidebar.subheader("Zone Selector")
server_filter = st.sidebar.selectbox(
"Selecione o escopo:",
options=[
"Consolidated View (Geral)",
"Azure Infrastructure Zone",
"OCI Infrastructure Zone"
]
)
# File Ingestion
st.sidebar.subheader("Log Ingestion")
uploaded_file = st.sidebar.file_uploader(
"Importar relatório NetBackup (CSV):",
type=["csv"],
help="Arraste e solte o CSV extraído do Veritas NetBackup"
)
if uploaded_file is not None:
try:
# Read content and compute hash for deduplication logic
file_bytes = uploaded_file.read()
file_hash = compute_hash(file_bytes)
# Check if already processed in database
is_processed = db.is_file_processed(file_hash)
# Parse CSV
df_parsed = parse_nbu_csv(file_bytes)
if not df_parsed.empty:
# Perform SQLite UPSERT operation for each parsed job
db.save_jobs(df_parsed)
if not is_processed:
db.mark_file_processed(file_hash, uploaded_file.name)
st.toast(f"Relatório '{uploaded_file.name}' importado e sincronizado no banco!", icon="✅")
else:
st.info(f"O relatório '{uploaded_file.name}' já foi importado anteriormente. Os dados foram atualizados no banco.")
else:
st.error("O arquivo fornecido está vazio ou mal formatado.")
except Exception as e:
st.error(f"Erro ao processar arquivo: {str(e)}")
# Demo data load button
db_jobs = db.get_historical_jobs()
if not db_jobs:
st.sidebar.markdown("---")
st.sidebar.write("Sem registros no banco. Deseja carregar dados simulados?")
if st.sidebar.button("Carregar Dados de Demonstração"):
demo_data = """###################### TABLE (Job Summary)####################
Job ID,Client,Policy,Type,Exit Code,Start Time,Finish Time,Duration,MBytes,# of Files,Primary Server,Media Server
10001,srv-web-01,Prod_Web_Apps,Backup,0,2026-07-13 01:00:00,2026-07-13 01:15:23,00:15:23,"1,540.50",45612,srvpalcvnbu01.elo.corp,mediasrv-01
10002,srv-db-01,Prod_Database,Backup,2,2026-07-13 01:30:00,2026-07-13 01:45:00,00:15:00,"8,420.00",12500,srvpalcvnbu01.elo.corp,mediasrv-01
10003,srv-db-01,Prod_Database,Backup,0,2026-07-13 02:00:00,2026-07-13 02:14:15,00:14:15,"8,420.00",12500,srvpalcvnbu01.elo.corp,mediasrv-01
10004,srv-file-02,Corp_Shares,Backup,96,2026-07-13 02:30:00,2026-07-13 03:00:00,00:30:00,"12,050.40",154210,srvpalcocinbupri01.elo.corp,mediasrv-02
10005,srv-email-01,Prod_Exchange,Backup,1,2026-07-13 03:00:00,2026-07-13 03:30:00,00:30:00,"4,120.30",8912,srvpalcvnbu01.elo.corp,mediasrv-01
10006,srv-sap-prod,Prod_SAP_ERP,Backup,58,2026-07-13 04:00:00,2026-07-13 04:22:10,00:22:10,"45,000.00",301244,srvpalcocinbupri01.elo.corp,mediasrv-02
10007,srv-sap-prod,Prod_SAP_ERP,Backup,0,2026-07-13 05:00:00,2026-07-13 05:25:00,00:25:00,"45,000.00",301244,srvpalcocinbupri01.elo.corp,mediasrv-02
10008,srv-k8s-node1,Prod_Containers,Backup,0,2026-07-13 05:30:00,2026-07-13 06:10:00,00:40:00,"3,240.10",56124,srvpalcvnbu01.elo.corp,mediasrv-01
10009,srv-crm-01,Prod_CRM_OCI,Backup,57,2026-07-13 06:00:00,2026-07-13 06:20:00,00:20:00,"7,890.00",94125,srvpalcocinbupri01.elo.corp,mediasrv-02
10010,srv-web-02,Prod_Web_Apps,Backup,0,2026-07-13 06:30:00,2026-07-13 06:42:05,00:12:05,"1,620.40",47120,srvpalcvnbu01.elo.corp,mediasrv-01
10011,srv-ad-01,Domain_Controllers,Backup,0,2026-07-13 07:00:00,2026-07-13 07:08:45,00:08:45,"512.00",4102,srvpalcvnbu01.elo.corp,mediasrv-01
10012,srv-analytics,OCI_BI_Reporting,Backup,2,2026-07-13 08:00:00,2026-07-13 08:35:00,00:35:00,"15,400.00",85124,srvpalcocinbupri01.elo.corp,mediasrv-02
"""
df_parsed = parse_nbu_csv(demo_data)
db.save_jobs(df_parsed)
st.rerun()
# Apply logic and query historical database records
db_jobs = db.get_historical_jobs()
df = pd.DataFrame(db_jobs)
df_filtered = pd.DataFrame()
if not df.empty:
# Set proper datetime types for sorting/filtering
df['start_time'] = pd.to_datetime(df['start_time'])
df['finish_time'] = pd.to_datetime(df['finish_time'])
# Filter by Cloud Infrastructure Zone
if server_filter == "Azure Infrastructure Zone":
df_filtered = df[df['primary_server'] == 'srvpalcvnbu01.elo.corp']
elif server_filter == "OCI Infrastructure Zone":
df_filtered = df[df['primary_server'] == 'srvpalcocinbupri01.elo.corp']
else:
df_filtered = df
# Application Title
st.markdown("<h1 style='margin-bottom: 25px;'>NetBackup Log Insights & Mitigation Tracker</h1>", unsafe_allow_html=True)
if df.empty:
# Beautiful empty state welcome page
st.info("👋 Bem-vindo! Carregue um arquivo de log do NetBackup (CSV) na barra lateral esquerda ou clique em 'Carregar Dados de Demonstração' para preencher o banco de dados persistente.")
col1, col2 = st.columns(2)
with col1:
st.markdown("""
### Arquitetura de Containers (v2.0)
* 🐳 **VPS Deploy Ready:** Totalmente empacotado para execução em Docker e orquestração Docker Compose.
* 💾 **Persistência SQLite:** Banco persistido no volume `/app/data/nbu_insights.db`.
* 🔄 **UPSERT Engine:** Ingestão incremental garantida baseado na chave `(Job ID)`.
* 📊 **Deduplicação Inteligente:** Identifica falhas transitórias com correções automáticas em reexecuções de logs posteriores.
""")
with col2:
st.markdown("""
### Relatórios Mitigados em PDF
* 📄 **PDF Export Engine:** Integração com biblioteca `fpdf2`.
* 📥 **Exportação Rápida:** Gera relatórios em A4 contendo resumos operacionais e o histórico completo de notas técnicas inseridas por engenheiros.
""")
else:
# Tabs layout
tab_dashboard, tab_table, tab_mitigation = st.tabs([
"📊 Dashboard de Performance",
"📋 Tabela de Execuções",
"🛠️ Plano de Ações & Mitigações"
])
# Tab 1: Dashboard
with tab_dashboard:
# Calculate dashboard metrics
total_jobs = len(df_filtered)
success_jobs = len(df_filtered[df_filtered['exit_code'] <= 1])
success_rate = (success_jobs / total_jobs * 100) if total_jobs > 0 else 0.0
# Space savings
total_pre_dedup = df_filtered['mbytes'].sum()
import hashlib
# Calculate simulated post-deduplicated sizes
total_post_dedup = 0.0
for _, r in df_filtered.iterrows():
total_post_dedup += r['mbytes'] * (0.15 + (int(hashlib.md5(str(r['job_id']).encode()).hexdigest(), 16) % 11) / 100.0)
dedup_ratio = (total_pre_dedup / total_post_dedup) if total_post_dedup > 0.0 else 1.0
space_saved = ((1 - (total_post_dedup / total_pre_dedup)) * 100) if total_pre_dedup > 0.0 else 0.0
# Active failures (Exit Code > 1 AND is_rerun_success == 0 AND status != 'Resolvido')
active_failures = len(df_filtered[
(df_filtered['exit_code'] > 1) &
(df_filtered['is_rerun_success'] == 0) &
(df_filtered['status'] != 'Resolvido')
])
# Render top KPI metrics row
col_metric1, col_metric2, col_metric3 = st.columns(3)
with col_metric1:
st.metric(
label="Global Success Ratio",
value=f"{success_rate:.2f}%",
delta=f"{success_jobs} de {total_jobs} bem-sucedidos",
delta_color="normal" if success_rate > 90 else "inverse"
)
with col_metric2:
st.metric(
label="Deduplication Ratio (MSDP)",
value=f"{dedup_ratio:.2f}:1",
delta=f"{space_saved:.1f}% de economia de espaço"
)
with col_metric3:
st.metric(
label="Active Incidents Monitor",
value=f"{active_failures}",
delta="Requer atenção operacional" if active_failures > 0 else "Operação normalizada",
delta_color="inverse" if active_failures > 0 else "normal"
)
# PDF Generation action button
st.markdown("---")
pdf_bytes = rg.generate_mitigation_pdf(df_filtered.to_dict('records'), server_filter)
st.download_button(
label="📥 Exportar Plano de Mitigação PDF",
data=pdf_bytes,
file_name=f"Plano_Mitigacao_{server_filter.replace(' ', '_')}.pdf",
mime="application/pdf"
)
# Distribution charts
st.markdown("<h3 style='margin-top:25px;'>Visualizações Operacionais</h3>", unsafe_allow_html=True)
col_chart1, col_chart2 = st.columns(2)
with col_chart1:
# Rerun / Fail categories pie chart
categories = []
counts = []
colors = []
# Successes
suc_count = len(df_filtered[df_filtered['exit_code'] <= 1])
if suc_count > 0:
categories.append("Sucesso (Exit 0/1)")
counts.append(suc_count)
colors.append("#00C853")
# Failures Reexecuted
reex_count = len(df_filtered[(df_filtered['exit_code'] > 1) & (df_filtered['is_rerun_success'] == 1)])
if reex_count > 0:
categories.append("Falha Reexecutada")
counts.append(reex_count)
colors.append("#00D2FF")
# Failures Mitigated
mit_count = len(df_filtered[
(df_filtered['exit_code'] > 1) &
(df_filtered['is_rerun_success'] == 0) &
(df_filtered['status'] == 'Resolvido')
])
if mit_count > 0:
categories.append("Falha Mitigada (DB)")
counts.append(mit_count)
colors.append("#FFAB00")
# Active Failures
act_count = len(df_filtered[
(df_filtered['exit_code'] > 1) &
(df_filtered['is_rerun_success'] == 0) &
(df_filtered['status'] != 'Resolvido')
])
if act_count > 0:
categories.append("Falha Ativa")
counts.append(act_count)
colors.append("#FF4B4B")
if categories:
fig_donut = go.Figure(data=[go.Pie(
labels=categories,
values=counts,
hole=.4,
marker=dict(colors=colors, line=dict(color='#0B0F19', width=2))
)])
fig_donut.update_layout(
title_text="Distribuição de Status de Backup",
paper_bgcolor='rgba(0,0,0,0)',
plot_bgcolor='rgba(0,0,0,0)',
font_color='#E2E8F0',
legend=dict(orientation="h", y=-0.1)
)
st.plotly_chart(fig_donut, use_container_width=True)
with col_chart2:
# Volume written by client
if not df_filtered.empty:
df_grouped = df_filtered.groupby('client')['mbytes'].sum().reset_index()
df_grouped = df_grouped.sort_values(by='mbytes', ascending=False).head(7)
fig_bar = go.Figure()
fig_bar.add_trace(go.Bar(
x=df_grouped['client'],
y=df_grouped['mbytes'],
name='Pre-Deduplicated Size (MB)',
marker_color='#0052CC'
))
fig_bar.update_layout(
title_text="Tamanho do Backup Ingerido por Cliente (Top 7)",
paper_bgcolor='rgba(0,0,0,0)',
plot_bgcolor='rgba(0,0,0,0)',
font_color='#E2E8F0',
xaxis=dict(gridcolor='#1E293B'),
yaxis=dict(gridcolor='#1E293B')
)
st.plotly_chart(fig_bar, use_container_width=True)
# Tab 2: Job Table
with tab_table:
st.markdown("<h3 style='margin-bottom:15px;'>Lista Completa de Jobs Ingeridos</h3>", unsafe_allow_html=True)
# State Filter options
error_state_filter = st.selectbox(
"Visualização de Erros:",
options=["Todos os Registros", "Todos os Erros", "Erros Sem Tratativa / Pendentes", "Erros Corrigidos Automatizados (Reexecutados)"]
)
# Filter dataframe based on state selection
df_grid = df_filtered.copy()
if error_state_filter == "Todos os Erros":
df_grid = df_grid[df_grid['exit_code'] > 1]
elif error_state_filter == "Erros Sem Tratativa / Pendentes":
df_grid = df_grid[
(df_grid['exit_code'] > 1) &
(df_grid['is_rerun_success'] == 0) &
(df_grid['status'] != 'Resolvido')
]
elif error_state_filter == "Erros Corrigidos Automatizados (Reexecutados)":
df_grid = df_grid[
(df_grid['exit_code'] > 1) &
(df_grid['is_rerun_success'] == 1)
]
if df_grid.empty:
st.info("Nenhum registro corresponde ao filtro de erro selecionado.")
else:
# Build display columns
def get_rerun_badge(row):
code = row['exit_code']
reex = row['is_rerun_success']
status_db = row['status']
if code <= 1:
return "Sucesso"
elif reex == 1:
return "✓ Reexecutado com Sucesso"
else:
return f"Falha (Mitigação: {status_db})"
df_grid['Indicador Visual'] = df_grid.apply(get_rerun_badge, axis=1)
df_grid['Start Time'] = df_grid['start_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
df_grid['Finish Time'] = df_grid['finish_time'].dt.strftime('%Y-%m-%d %H:%M:%S')
df_grid_display = df_grid[[
'job_id', 'primary_server', 'client', 'policy', 'type', 'exit_code',
'Indicador Visual', 'Start Time', 'Finish Time', 'duration_secs',
'mbytes', 'files_count', 'status', 'action_taken'
]]
st.dataframe(
df_grid_display,
use_container_width=True,
column_config={
"job_id": st.column_config.NumberColumn(format="%d"),
"duration_secs": st.column_config.NumberColumn(format="%d s"),
"mbytes": st.column_config.NumberColumn(format="%.2f MB"),
"files_count": st.column_config.NumberColumn(format="%d")
},
hide_index=True
)
# Tab 3: Mitigation Register CRUD
with tab_mitigation:
st.markdown("<h3 style='margin-bottom:15px;'>Registro de Ações Corretivas</h3>", unsafe_allow_html=True)
# Failed jobs
failed_jobs_df = df_filtered[df_filtered['exit_code'] > 1]
if failed_jobs_df.empty:
st.success("🎉 Nenhuma falha de backup identificada no escopo selecionado!")
else:
col_list, col_form = st.columns([1, 1])
with col_list:
st.markdown("#### Lista de Ocorrências com Erro")
failures_list = []
for _, row in failed_jobs_df.iterrows():
jid = int(row['job_id'])
reex = row['is_rerun_success']
db_status = row['status']
if reex == 1:
display_status = "Resolvido (Reexecutado)"
else:
display_status = f"Operação: {db_status}"
failures_list.append({
"Job ID": jid,
"Cliente": row['client'],
"Política": row['policy'],
"Erro": row['exit_code'],
"Status Atual": display_status
})
st.dataframe(pd.DataFrame(failures_list), use_container_width=True, hide_index=True)
with col_form:
st.markdown("#### Formulário de Mitigação")
# Dropdown option
job_options = [
f"{row['job_id']} - {row['client']} ({row['policy']})"
for _, row in failed_jobs_df.iterrows()
]
selected_job_option = st.selectbox(
"Selecione o Job com falha para atualizar:",
options=job_options
)
if selected_job_option:
selected_job_id = int(selected_job_option.split(" - ")[0])
job_row = failed_jobs_df[failed_jobs_df['job_id'] == selected_job_id].iloc[0]
current_text = job_row['action_taken'] if pd.notna(job_row['action_taken']) else ''
current_status = job_row['status'] if pd.notna(job_row['status']) else 'Pendente'
st.markdown(f"""
* **Servidor Master:** `{job_row['primary_server']}`
* **Código de Erro:** `Exit Code {job_row['exit_code']}`
* **Reexecutado com Sucesso?** `{"Sim" if job_row['is_rerun_success'] == 1 else "Não"}`
""")
with st.form(key="mitigation_form_v2", clear_on_submit=False):
action_text = st.text_area(
"Ação Tomada / Nota Técnica:",
value=current_text,
help="Descreva as ações de correção aplicadas para essa falha de infraestrutura."
)
status_option = st.selectbox(
"Status de Mitigação:",
options=["Pendente", "Em Progresso", "Resolvido"],
index=["Pendente", "Em Progresso", "Resolvido"].index(current_status)
)
submit_btn = st.form_submit_button("Salvar Registro")
if submit_btn:
db.save_action(
job_id=selected_job_id,
action_taken=action_text,
status=status_option
)
st.toast("Mitigação registrada e salva no banco!", icon="💾")
time.sleep(0.8)
st.rerun()