!wget https://webs.um.es/slopez/datasets/BigData/News_Category_Dataset_HuffPost.json

--2026-05-20 11:54:07--  https://webs.um.es/slopez/datasets/BigData/News_Category_Dataset_HuffPost.json
Resolving webs.um.es (webs.um.es)... 155.54.212.112
Connecting to webs.um.es (webs.um.es)|155.54.212.112|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 87295572 (83M) [application/json]
Saving to: ‘News_Category_Dataset_HuffPost.json’

News_Category_Datas 100%[===================>]  83.25M  84.1MB/s    in 1.0s    

2026-05-20 11:54:08 (84.1 MB/s) - ‘News_Category_Dataset_HuffPost.json’ saved [87295572/87295572]

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import re
from collections import Counter

# El dataset de HuffPost está representado en formato JSON por líneas.
df = pd.read_json('News_Category_Dataset_HuffPost.json', lines=True)

print("Dimensiones del dataset:", df.shape)
df.head()

Dimensiones del dataset: (209527, 6)

df.info()

<class 'pandas.DataFrame'>
RangeIndex: 209527 entries, 0 to 209526
Data columns (total 6 columns):
 #   Column             Non-Null Count   Dtype         
---  ------             --------------   -----         
 0   link               209527 non-null  str           
 1   headline           209527 non-null  str           
 2   category           209527 non-null  str           
 3   short_description  209527 non-null  str           
 4   authors            209527 non-null  str           
 5   date               209527 non-null  datetime64[us]
dtypes: datetime64[us](1), str(5)
memory usage: 9.6 MB

print("Ejemplo de noticia:")
print("- Link: ", df.iloc[0]["link"])
print("- Headline: ", df.iloc[0]["headline"])
print("- Category: ", df.iloc[0]["category"])
print("- Short description: ", df.iloc[0]["short_description"])
print("- Authors: ", df.iloc[0]["authors"])
print("- Date: ", df.iloc[0]["date"])

Ejemplo de noticia:
- Link:  https://www.huffpost.com/entry/covid-boosters-uptake-us_n_632d719ee4b087fae6feaac9
- Headline:  Over 4 Million Americans Roll Up Sleeves For Omicron-Targeted COVID Boosters
- Category:  U.S. NEWS
- Short description:  Health experts said it is too early to predict whether demand would match up with the 171 million doses of the new boosters the U.S. ordered for the fall.
- Authors:  Carla K. Johnson, AP
- Date:  2022-09-23 00:00:00

# Listado categorías en la columna "category"
df["category"].unique()

<StringArray>
[     'U.S. NEWS',         'COMEDY',      'PARENTING',     'WORLD NEWS',
 'CULTURE & ARTS',           'TECH',         'SPORTS',  'ENTERTAINMENT',
       'POLITICS',     'WEIRD NEWS',    'ENVIRONMENT',      'EDUCATION',
          'CRIME',        'SCIENCE',       'WELLNESS',       'BUSINESS',
 'STYLE & BEAUTY',   'FOOD & DRINK',          'MEDIA',   'QUEER VOICES',
  'HOME & LIVING',          'WOMEN',   'BLACK VOICES',         'TRAVEL',
          'MONEY',       'RELIGION',  'LATINO VOICES',         'IMPACT',
       'WEDDINGS',        'COLLEGE',        'PARENTS', 'ARTS & CULTURE',
          'STYLE',          'GREEN',          'TASTE', 'HEALTHY LIVING',
  'THE WORLDPOST',      'GOOD NEWS',      'WORLDPOST',          'FIFTY',
           'ARTS',        'DIVORCE']
Length: 42, dtype: str

# Número de documentos por categoría
conteo_categorias = df["category"].value_counts()

print("Número de categorías:", conteo_categorias.shape[0])
conteo_categorias.head(20)

Número de categorías: 42

category
POLITICS          35602
WELLNESS          17945
ENTERTAINMENT     17362
TRAVEL             9900
STYLE & BEAUTY     9814
PARENTING          8791
HEALTHY LIVING     6694
QUEER VOICES       6347
FOOD & DRINK       6340
BUSINESS           5992
COMEDY             5400
SPORTS             5077
BLACK VOICES       4583
HOME & LIVING      4320
PARENTS            3955
THE WORLDPOST      3664
WEDDINGS           3653
WOMEN              3572
CRIME              3562
IMPACT             3484
Name: count, dtype: int64

# Visualización de las categorías más frecuentes
top_categorias = conteo_categorias.head(15)

plt.figure(figsize=(10, 6))
top_categorias.sort_values().plot(kind="barh")
plt.title("Categorías más frecuentes del dataset")
plt.xlabel("Número de documentos")
plt.ylabel("Categoría")
plt.show()

# Categorías propuestas para la sesión guiada.
categorias_propuestas = [
    "POLITICS",
    "SPORTS",
    "BUSINESS",
    "TRAVEL",
    "FOOD & DRINK",
    "WELLNESS",
    "ENTERTAINMENT",
    "TECH"
]

categorias_disponibles = []
for categoria in categorias_propuestas:
    if categoria in df["category"].unique():
        categorias_disponibles.append(categoria)

print("Categorías seleccionadas disponibles:")
print(categorias_disponibles)

Categorías seleccionadas disponibles:
['POLITICS', 'SPORTS', 'BUSINESS', 'TRAVEL', 'FOOD & DRINK', 'WELLNESS', 'ENTERTAINMENT', 'TECH']

# Filtramos el dataset para quedarnos con las categorías seleccionadas
df_filtrado = df[df["category"].isin(categorias_disponibles)].copy()

print("Dimensiones tras filtrar categorías:", df_filtrado.shape)
df_filtrado["category"].value_counts()

Dimensiones tras filtrar categorías: (100322, 6)

category
POLITICS         35602
WELLNESS         17945
ENTERTAINMENT    17362
TRAVEL            9900
FOOD & DRINK      6340
BUSINESS          5992
SPORTS            5077
TECH              2104
Name: count, dtype: int64

print(df_filtrado.columns)

Index(['link', 'headline', 'category', 'short_description', 'authors', 'date'], dtype='str')

# Número máximo de documentos por categoría.
# Podemos aumentar este valor si el entorno de ejecución lo permite.
"""
max_documentos_por_categoria = 500

df_reducido = (
    df_filtrado
    .groupby("category", group_keys=False)
    .apply(lambda datos_categoria: datos_categoria.sample(
        n=min(len(datos_categoria), max_documentos_por_categoria),
        random_state=42
    ))
    .reset_index(drop=True)
)

print("Dimensiones del corpus reducido:", df_reducido.shape)
df_reducido["category"].value_counts()
"""

# Número máximo de documentos por categoría.
# Podemos aumentar este valor si el entorno de ejecución lo permite.
max_documentos_por_categoria: int = 500

# Creamos una lista para almacenar los subconjuntos de cada categoría.
subconjuntos: list = []

# Recorremos cada categoría y sus documentos asociados.
for categoria, datos_categoria in df_filtrado.groupby("category"):

    # Calculamos cuántos documentos vamos a tomar de esta categoría.
    # Si hay menos de 500, tomamos todos.
    numero_documentos: int = min(
        len(datos_categoria),
        max_documentos_por_categoria
    )

    # Seleccionamos aleatoriamente documentos de esta categoría.
    datos_categoria_reducidos = datos_categoria.sample(
        n=numero_documentos,
        random_state=42
    )

    # Añadimos el subconjunto a la lista.
    subconjuntos.append(datos_categoria_reducidos)

# Unimos todos los subconjuntos en un único DataFrame.
df_reducido = pd.concat(subconjuntos, ignore_index=True)

print("Dimensiones del corpus reducido:", df_reducido.shape)
print(df_reducido["category"].value_counts())

Dimensiones del corpus reducido: (4000, 6)
category
BUSINESS         500
ENTERTAINMENT    500
FOOD & DRINK     500
POLITICS         500
SPORTS           500
TECH             500
TRAVEL           500
WELLNESS         500
Name: count, dtype: int64

# Sustituimos valores ausentes por cadenas vacías
df_reducido["headline"] = df_reducido["headline"].fillna("")
df_reducido["short_description"] = df_reducido["short_description"].fillna("")

# Creamos una columna de texto combinando titular y descripción
df_reducido["text"] = (
    df_reducido["headline"].astype(str)
    + ". "
    + df_reducido["short_description"].astype(str)
)

# Eliminamos espacios sobrantes
df_reducido["text"] = df_reducido["text"].str.strip()

df_reducido[["category", "headline", "short_description", "text"]].head()

print("Ejemplo de noticia:")
print("- Headline: ", df_reducido.iloc[0]["headline"])
print("- Short description: ", df_reducido.iloc[0]["short_description"])
print("- Text: ", df_reducido.iloc[0]["text"])

Ejemplo de noticia:
- Headline:  How to Manage Your Personal Brand
- Short description:  Make no mistake: If you have a Facebook account, an Instagram page, a Twitter profile, you are a brand. Every time you upload a photo, add a link, or post an update, you're putting into the world another idea of yourself and what you stand for.
- Text:  How to Manage Your Personal Brand. Make no mistake: If you have a Facebook account, an Instagram page, a Twitter profile, you are a brand. Every time you upload a photo, add a link, or post an update, you're putting into the world another idea of yourself and what you stand for.

# Calculamos una longitud simple en número de caracteres
df_reducido["text_length"] = df_reducido["text"].str.len()

print(df_reducido["text_length"].describe())

plt.figure(figsize=(8, 5))
plt.hist(df_reducido["text_length"], bins=40)
plt.title("Distribución de la longitud de los textos")
plt.xlabel("Longitud en caracteres")
plt.ylabel("Número de documentos")
plt.show()

count    4000.00000
mean      171.04225
std        75.56621
min        14.00000
25%       119.00000
50%       166.00000
75%       206.00000
max      1098.00000
Name: text_length, dtype: float64

# Eliminamos textos demasiado cortos
longitud_minima = 40

df_reducido = df_reducido[df_reducido["text_length"] >= longitud_minima].copy()
df_reducido = df_reducido.reset_index(drop=True)

print("Dimensiones tras eliminar textos cortos:", df_reducido.shape)

Dimensiones tras eliminar textos cortos: (3952, 8)

import nltk

# Descargamos la lista de stopwords de NLTK.
# Si ya está descargada, NLTK no la vuelve a descargar.
nltk.download("stopwords")

[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/sergio/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!

True

from nltk.corpus import stopwords

stopwords_ingles = set(stopwords.words("english"))

# Añadimos algunas palabras frecuentes poco informativas en noticias.
# Esta lista puede ajustarse tras inspeccionar resultados.
stopwords_adicionales = {
    "said", "says", "say", "also", "one", "two", "new", "would", "could",
    "get", "like", "time", "people", "year", "years", "make", "made"
}

stopwords_totales = stopwords_ingles.union(stopwords_adicionales)

print("Número total de stopwords:", len(stopwords_totales))

Número total de stopwords: 215

stopwords_totales

{'a',
 'about',
 'above',
 'after',
 'again',
 'against',
 'ain',
 'all',
 'also',
 'am',
 'an',
 'and',
 'any',
 'are',
 'aren',
 "aren't",
 'as',
 'at',
 'be',
 'because',
 'been',
 'before',
 'being',
 'below',
 'between',
 'both',
 'but',
 'by',
 'can',
 'could',
 'couldn',
 "couldn't",
 'd',
 'did',
 'didn',
 "didn't",
 'do',
 'does',
 'doesn',
 "doesn't",
 'doing',
 'don',
 "don't",
 'down',
 'during',
 'each',
 'few',
 'for',
 'from',
 'further',
 'get',
 'had',
 'hadn',
 "hadn't",
 'has',
 'hasn',
 "hasn't",
 'have',
 'haven',
 "haven't",
 'having',
 'he',
 "he'd",
 "he'll",
 "he's",
 'her',
 'here',
 'hers',
 'herself',
 'him',
 'himself',
 'his',
 'how',
 'i',
 "i'd",
 "i'll",
 "i'm",
 "i've",
 'if',
 'in',
 'into',
 'is',
 'isn',
 "isn't",
 'it',
 "it'd",
 "it'll",
 "it's",
 'its',
 'itself',
 'just',
 'like',
 'll',
 'm',
 'ma',
 'made',
 'make',
 'me',
 'mightn',
 "mightn't",
 'more',
 'most',
 'mustn',
 "mustn't",
 'my',
 'myself',
 'needn',
 "needn't",
 'new',
 'no',
 'nor',
 'not',
 'now',
 'o',
 'of',
 'off',
 'on',
 'once',
 'one',
 'only',
 'or',
 'other',
 'our',
 'ours',
 'ourselves',
 'out',
 'over',
 'own',
 'people',
 're',
 's',
 'said',
 'same',
 'say',
 'says',
 'shan',
 "shan't",
 'she',
 "she'd",
 "she'll",
 "she's",
 'should',
 "should've",
 'shouldn',
 "shouldn't",
 'so',
 'some',
 'such',
 't',
 'than',
 'that',
 "that'll",
 'the',
 'their',
 'theirs',
 'them',
 'themselves',
 'then',
 'there',
 'these',
 'they',
 "they'd",
 "they'll",
 "they're",
 "they've",
 'this',
 'those',
 'through',
 'time',
 'to',
 'too',
 'two',
 'under',
 'until',
 'up',
 've',
 'very',
 'was',
 'wasn',
 "wasn't",
 'we',
 "we'd",
 "we'll",
 "we're",
 "we've",
 'were',
 'weren',
 "weren't",
 'what',
 'when',
 'where',
 'which',
 'while',
 'who',
 'whom',
 'why',
 'will',
 'with',
 'won',
 "won't",
 'would',
 'wouldn',
 "wouldn't",
 'y',
 'year',
 'years',
 'you',
 "you'd",
 "you'll",
 "you're",
 "you've",
 'your',
 'yours',
 'yourself',
 'yourselves'}

def preprocesar_texto(texto):
    '''
    Aplica un preprocesamiento básico sobre un texto en inglés.

    Pasos:
    1. Convertir a minúsculas.
    2. Eliminar caracteres no alfabéticos.
    3. Separar en tokens.
    4. Eliminar stopwords.
    5. Eliminar tokens muy cortos.

    Devuelve:
    - Una cadena con los tokens procesados unidos por espacios.
    '''

    # 1. Convertir a minúsculas
    texto = texto.lower()
    # 2. Eliminar caracteres no alfabéticos (también descarta números). \s significa un "espacio en blanco"
    texto = re.sub(r"[^a-z\s]", " ", texto)
    # 3. Separar en tokens, obviando espacios duplicados
    tokens = texto.split()

    # Filtramos aquellos tokens que no sean stopwords y cuya longitud (nº de letras) sea mayor o igual a 3
    tokens_filtrados = []
    for token in tokens:
        if token not in stopwords_totales and len(token) >= 3:
            tokens_filtrados.append(token)

    texto_procesado = " ".join(tokens_filtrados)

    return texto_procesado

df_reducido["processed_text"] = df_reducido["text"].apply(preprocesar_texto)

df_reducido[["category", "text", "processed_text"]].head()

for indice in range(3):
    print("=" * 100)
    print("Categoría:", df_reducido.loc[indice, "category"])
    print("\nTexto original:")
    print(df_reducido.loc[indice, "text"])
    print("\nTexto procesado:")
    print(df_reducido.loc[indice, "processed_text"])
    print()

====================================================================================================
Categoría: BUSINESS

Texto original:
How to Manage Your Personal Brand. Make no mistake: If you have a Facebook account, an Instagram page, a Twitter profile, you are a brand. Every time you upload a photo, add a link, or post an update, you're putting into the world another idea of yourself and what you stand for.

Texto procesado:
manage personal brand mistake facebook account instagram page twitter profile brand every upload photo add link post update putting world another idea stand

====================================================================================================
Categoría: BUSINESS

Texto original:
It Looks Like Uber's Winning Its War With New York. Grab the popcorn.

Texto procesado:
looks uber winning war york grab popcorn

====================================================================================================
Categoría: BUSINESS

Texto original:
The Progressive Promise of Today's Technology. A digital policy for the new century, tailored not just to the moment but for the future, is vital if we are to unleash economic growth, shared prosperity, and the full potential of technology for citizens and consumers. But such a policy architecture requires a new consensus -- on privacy, on security, on customer protections, on growth and mobility.

Texto procesado:
progressive promise today technology digital policy century tailored moment future vital unleash economic growth shared prosperity full potential technology citizens consumers policy architecture requires consensus privacy security customer protections growth mobility

# Unimos todos los textos procesados
todos_los_tokens = []

for texto in df_reducido["processed_text"]:
    tokens = texto.split()
    for token in tokens:
        todos_los_tokens.append(token)

# Calculamos el número de apariciones de cada token
frecuencias = Counter(todos_los_tokens)

# Obtenemos y mostramos las frecuencias de las 30 palabras más frecuentes
palabras_frecuentes = pd.DataFrame(
    frecuencias.most_common(30),
    columns=["word", "frequency"]
)

palabras_frecuentes

plt.figure(figsize=(10, 6))
plt.barh(
    palabras_frecuentes["word"][::-1],
    palabras_frecuentes["frequency"][::-1]
)
plt.title("Palabras más frecuentes tras el preprocesamiento")
plt.xlabel("Frecuencia")
plt.ylabel("Palabra")
plt.show()

df_reducido["processed_length"] = df_reducido["processed_text"].apply(lambda texto: len(texto.split()))

print(df_reducido["processed_length"].describe())

plt.figure(figsize=(8, 5))
plt.hist(df_reducido["processed_length"], bins=30)
plt.title("Longitud de los documentos procesados")
plt.xlabel("Número de tokens")
plt.ylabel("Número de documentos")
plt.show()

count    3952.000000
mean       16.234818
std         6.553792
min         3.000000
25%        12.000000
50%        16.000000
75%        20.000000
max        98.000000
Name: processed_length, dtype: float64

# Eliminamos documentos que hayan quedado con muy pocos tokens tras el preprocesamiento
min_tokens = 5

df_reducido = df_reducido[df_reducido["processed_length"] >= min_tokens].copy()
df_reducido = df_reducido.reset_index(drop=True)

print("Dimensiones tras filtrar por longitud procesada:", df_reducido.shape)

Dimensiones tras filtrar por longitud procesada: (3924, 10)

from sklearn.feature_extraction.text import CountVectorizer

# CountVectorizer construye automáticamente el vocabulario y la matriz documento-término.
# Como ya hemos preprocesado el texto, indicamos lowercase=False.
vectorizador_conteos = CountVectorizer(
    lowercase=False,
    min_df=5,
    max_df=0.80,
    max_features=3000
)

matriz_conteos = vectorizador_conteos.fit_transform(df_reducido["processed_text"])

print("Dimensiones de la matriz documento-término (nº documentos, nº términos):")
print(matriz_conteos.shape)

Dimensiones de la matriz documento-término (nº documentos, nº términos):
(3924, 2797)

vocabulario = vectorizador_conteos.get_feature_names_out()

print("Tamaño del vocabulario:", len(vocabulario))
print("Primeros términos del vocabulario:")
print(vocabulario[:50])

Tamaño del vocabulario: 2797
Primeros términos del vocabulario:
['abc' 'ability' 'able' 'abortion' 'absolutely' 'abuse' 'academy'
 'accepted' 'access' 'accessible' 'accident' 'according' 'account'
 'accusations' 'accused' 'across' 'act' 'acting' 'action' 'actions'
 'active' 'activities' 'activity' 'actor' 'actors' 'actress' 'actual'
 'actually' 'add' 'added' 'adding' 'addition' 'address' 'adds'
 'administration' 'admits' 'admitted' 'adorable' 'ads' 'adult' 'adults'
 'advance' 'advanced' 'advances' 'advantage' 'adventure' 'advertising'
 'advice' 'adviser' 'affect']

# Podemos modificar este valor para indicar el índice del documento concreto a visualizar
indice_documento = 0

# Extrae la fila de la matriz asociada al documento "indice_documento"
vector_documento = matriz_conteos[indice_documento].toarray()[0]

# Lista de términos del documento
terminos_presentes = []

# Recorre el vector del documento posición a posición, y añade el término a "terminos_presentes" si la frecuencia es > 0
# como una tupla junto con su frecuencia
# Ejemplo: ("market", 2)
for indice_termino, frecuencia in enumerate(vector_documento):
    if frecuencia > 0:
        terminos_presentes.append((vocabulario[indice_termino], frecuencia))

# Convierte la lista de términos presentes en un dataframe de pandas, con columnas "term" y "count", de forma ordenada por frecuencia
df_vector_documento = pd.DataFrame(
    terminos_presentes,
    columns=["term", "count"]
).sort_values(by="count", ascending=False)

print("Categoría real:", df_reducido.loc[indice_documento, "category"])
print("Texto original:")
print(df_reducido.loc[indice_documento, "text"])
print("\nTérminos presentes en la representación:")
df_vector_documento.head(20)

Categoría real: BUSINESS
Texto original:
How to Manage Your Personal Brand. Make no mistake: If you have a Facebook account, an Instagram page, a Twitter profile, you are a brand. Every time you upload a photo, add a link, or post an update, you're putting into the world another idea of yourself and what you stand for.

Términos presentes en la representación:

from sklearn.decomposition import LatentDirichletAllocation

numero_topicos = 8

modelo_lda = LatentDirichletAllocation(
    n_components=numero_topicos,
    random_state=42,
    learning_method="batch",
    max_iter=20
)

modelo_lda.fit(matriz_conteos)

print("Modelo LDA entrenado")

Modelo LDA entrenado

def obtener_palabras_topico(modelo, vocabulario, numero_palabras=10):
    '''
    Devuelve una tabla con las palabras principales de cada tópico.
    '''
    filas = []

    for indice_topico, distribucion_palabras in enumerate(modelo.components_):
        indices_ordenados = distribucion_palabras.argsort()[::-1]
        indices_top = indices_ordenados[:numero_palabras]

        palabras = []
        pesos = []

        for indice in indices_top:
            palabras.append(vocabulario[indice])
            pesos.append(distribucion_palabras[indice])

        filas.append({
            "topic": indice_topico,
            "top_words": ", ".join(palabras),
            "weights": pesos
        })

    return pd.DataFrame(filas)

tabla_topicos = obtener_palabras_topico(modelo_lda, vocabulario, numero_palabras=12)
tabla_topicos[["topic", "top_words"]]

categorias_propuestas = [
    "POLITICS",
    "SPORTS",
    "BUSINESS",
    "TRAVEL",
    "FOOD & DRINK",
    "WELLNESS",
    "ENTERTAINMENT",
    "TECH"
]

def graficar_topico(modelo, vocabulario, indice_topico, numero_palabras=10):
    distribucion_palabras = modelo.components_[indice_topico]
    indices_ordenados = distribucion_palabras.argsort()[::-1]
    indices_top = indices_ordenados[:numero_palabras]

    palabras = vocabulario[indices_top]
    pesos = distribucion_palabras[indices_top]

    plt.figure(figsize=(8, 5))
    plt.barh(palabras[::-1], pesos[::-1])
    plt.title(f"Palabras principales del tópico {indice_topico}")
    plt.xlabel("Peso")
    plt.ylabel("Palabra")
    plt.show()

# Ejemplo: mostramos el tópico 0
graficar_topico(modelo_lda, vocabulario, indice_topico=0, numero_palabras=10)

# Mostramos todos los tópicos uno a uno
for indice_topico in range(numero_topicos):
    graficar_topico(modelo_lda, vocabulario, indice_topico=indice_topico, numero_palabras=10)

# Transformamos la matriz de conteos para obtener la distribución de tópicos por documento
distribucion_documentos = modelo_lda.transform(matriz_conteos)

# Salida: matriz con forma (nº documentos, nº tópicos).
# Cada fila representa un documento.
# Cada columna representa un tópico.
# Cada celda contiene la proporción o peso de ese tópico en ese documento.
# Los valores de cada fila suman aproximadamente 1.
print("Dimensiones de la matriz documento-tópico:")
print(distribucion_documentos.shape)

Dimensiones de la matriz documento-tópico:
(3924, 8)

# Creamos columnas con el peso de cada tópico en cada documento

# Variables de partida:
# - df_reducido: información original y procesada de cada noticia
# - distribucion_documentos: distribución de tópicos que LDA ha estimado por documento

df_topicos_documentos = df_reducido[["category", "headline", "short_description", "text", "processed_text"]].copy()

# Para cada tópico crea una nueva columna topic_nº y la añade a los datos procesados de partida
for indice_topico in range(numero_topicos):
    nombre_columna = f"topic_{indice_topico}"
    df_topicos_documentos[nombre_columna] = distribucion_documentos[:, indice_topico]

# Añadimos dos columnas relacionadas con el tópico mayoritario (nº de tópico y su peso)
df_topicos_documentos["dominant_topic"] = distribucion_documentos.argmax(axis=1)
df_topicos_documentos["dominant_topic_weight"] = distribucion_documentos.max(axis=1)

df_topicos_documentos.head()

def mostrar_documentos_representativos(df_resultados, indice_topico, numero_documentos=5):
    columna_topico = f"topic_{indice_topico}"

    columnas_mostrar = [
        "category",
        "headline",
        "short_description",
        columna_topico,
        "dominant_topic",
        "dominant_topic_weight"
    ]

    # Ordena los documentos en base a la columna del tópico asociada al índice de tópico indicado como parámetro
    documentos = (
        df_resultados
        .sort_values(by=columna_topico, ascending=False)
        .head(numero_documentos)
    )

    return documentos[columnas_mostrar]

# Ejemplo: documentos más representativos del tópico 0
mostrar_documentos_representativos(df_topicos_documentos, indice_topico=0, numero_documentos=5)

# Revisamos documentos representativos de todos los tópicos
# Un for es un bucle que repite algo varias veces (en este caso, tantas como valores para numero_topicos tengamos)
for indice_topico in range(numero_topicos):
    print("=" * 120)
    print(f"TÓPICO {indice_topico}")
    print("- Palabras principales:")
    print(tabla_topicos.loc[tabla_topicos["topic"] == indice_topico, "top_words"].values[0])
    print("\n- Documentos representativos:")
    display(mostrar_documentos_representativos(df_topicos_documentos, indice_topico, numero_documentos=3))

========================================================================================================================
TÓPICO 0
- Palabras principales:
trump, donald, president, obama, clinton, house, court, gop, hillary, may, photos, republican

- Documentos representativos:

========================================================================================================================
TÓPICO 1
- Palabras principales:
first, star, life, way, uber, data, nfl, olympic, sports, night, best, world

- Documentos representativos:

========================================================================================================================
TÓPICO 2
- Palabras principales:
city, things, photos, york, best, study, every, back, business, world, getting, google

- Documentos representativos:

========================================================================================================================
TÓPICO 3
- Palabras principales:
apple, week, iphone, back, help, best, travel, life, ways, rumors, first, company

- Documentos representativos:

========================================================================================================================
TÓPICO 4
- Palabras principales:
food, photos, facebook, think, holiday, well, know, eat, weight, recipes, need, billion

- Documentos representativos:

========================================================================================================================
TÓPICO 5
- Palabras principales:
health, healthy, may, keep, take, want, good, game, cancer, companies, even, still

- Documentos representativos:

========================================================================================================================
TÓPICO 6
- Palabras principales:
world, day, first, state, little, best, play, stress, way, around, video, facebook

- Documentos representativos:

========================================================================================================================
TÓPICO 7
- Palabras principales:
food, super, day, watch, summer, week, bowl, videos, recipe, video, youtube, see

- Documentos representativos:

# Tabla de frecuencia entre categoría real y tópico dominante
# La función crosstab() cruza dos variables categóricas y cuenta cuántos casos hay en cada combinación
# En este caso, cuenta cuántos documentos de cada categoría original tienen cada tópico como tópico dominante
tabla_categoria_topico = pd.crosstab(
    df_topicos_documentos["category"],
    df_topicos_documentos["dominant_topic"]
)

tabla_categoria_topico

plt.figure(figsize=(10, 6))
plt.imshow(tabla_categoria_topico, aspect="auto")
plt.title("Categoría real frente a tópico dominante")
plt.xlabel("Tópico dominante")
plt.ylabel("Categoría real")
plt.xticks(
    ticks=np.arange(tabla_categoria_topico.shape[1]),
    labels=tabla_categoria_topico.columns
)
plt.yticks(
    ticks=np.arange(tabla_categoria_topico.shape[0]),
    labels=tabla_categoria_topico.index
)
plt.colorbar(label="Número de documentos")
plt.show()

def entrenar_y_resumir_lda(matriz, vocabulario, numero_topicos, numero_palabras=8):
    modelo = LatentDirichletAllocation(
        n_components=numero_topicos,
        random_state=42,
        learning_method="batch",
        max_iter=20
    )

    modelo.fit(matriz)

    tabla = obtener_palabras_topico(
        modelo,
        vocabulario,
        numero_palabras=numero_palabras
    )

    return modelo, tabla[["topic", "top_words"]]

valores_k = [4, 6, 8, 10]

resultados_modelos = {}

for k in valores_k:
    print("=" * 120)
    print(f"Modelo LDA con {k} tópicos")

    modelo_k, tabla_k = entrenar_y_resumir_lda(
        matriz_conteos,
        vocabulario,
        numero_topicos=k,
        numero_palabras=10
    )

    resultados_modelos[k] = {
        "model": modelo_k,
        "topics": tabla_k
    }

    display(tabla_k)

========================================================================================================================
Modelo LDA con 4 tópicos

========================================================================================================================
Modelo LDA con 6 tópicos

========================================================================================================================
Modelo LDA con 8 tópicos

========================================================================================================================
Modelo LDA con 10 tópicos

resultados_modelos

{4: {'model': LatentDirichletAllocation(max_iter=20, n_components=4, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, donald, president, food, may, know, oba...
  1      1  facebook, star, first, video, world, state, nf...
  2      2  life, world, google, day, food, best, business...
  3      3  week, apple, day, photos, want, top, watch, ip...},
 6: {'model': LatentDirichletAllocation(max_iter=20, n_components=6, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, clinton, better, way, may, know, even, ...
  1      1  star, first, video, state, night, uber, film, ...
  2      2  life, world, things, city, back, take, game, y...
  3      3  apple, week, iphone, travel, think, best, hous...
  4      4  food, photos, day, recipes, best, holiday, wor...
  5      5  trump, health, donald, president, even, good, ...},
 8: {'model': LatentDirichletAllocation(max_iter=20, n_components=8, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, donald, president, obama, clinton, hous...
  1      1  first, star, life, way, uber, data, nfl, olymp...
  2      2  city, things, photos, york, best, study, every...
  3      3  apple, week, iphone, back, help, best, travel,...
  4      4  food, photos, facebook, think, holiday, well, ...
  5      5  health, healthy, may, keep, take, want, good, ...
  6      6  world, day, first, state, little, best, play, ...
  7      7  food, super, day, watch, summer, week, bowl, v...},
 10: {'model': LatentDirichletAllocation(max_iter=20, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, donald, president, live, way, white, fi...
  1      1  first, olympic, sports, star, uber, fans, way,...
  2      2  things, photos, got, world, google, change, ga...
  3      3  travel, apple, nfl, best, think, still, want, ...
  4      4  know, social, want, facebook, job, news, media...
  5      5  health, healthy, take, let, eating, good, keep...
  6      6  day, world, twitter, facebook, first, sleep, c...
  7      7  food, recipes, photos, summer, week, day, reci...
  8      8  women, life, best, world, things, know, photos...
  9      9  trump, apple, clinton, court, week, donald, go...}}

# Construimos una nueva matriz documento-término usando unigramas y bigramas.
# En este caso, el vocabulario puede contener palabras individuales y pares de palabras consecutivas.

vectorizador_ngramas = CountVectorizer(
    lowercase=False,
    min_df=5,
    max_df=0.80,
    max_features=3000,
    ngram_range=(1, 2)
)

matriz_ngramas = vectorizador_ngramas.fit_transform(df_reducido["processed_text"])
vocabulario_ngramas = vectorizador_ngramas.get_feature_names_out()

print("Dimensiones de la matriz documento-término con unigramas y bigramas:")
print(matriz_ngramas.shape)

print("\nTamaño del vocabulario con n-gramas:", len(vocabulario_ngramas))

print("\nPrimeros términos del vocabulario:")
print(vocabulario_ngramas[:50])

Dimensiones de la matriz documento-término con unigramas y bigramas:
(3924, 2953)

Tamaño del vocabulario con n-gramas: 2953

Primeros términos del vocabulario:
['abc' 'ability' 'able' 'abortion' 'absolutely' 'abuse' 'academy'
 'accepted' 'access' 'accessible' 'accident' 'according' 'account'
 'accusations' 'accused' 'across' 'across country' 'act' 'acting' 'action'
 'actions' 'active' 'activities' 'activity' 'actor' 'actors' 'actress'
 'actual' 'actually' 'actually want' 'add' 'added' 'adding' 'addition'
 'address' 'adds' 'administration' 'admits' 'admitted' 'adorable' 'ads'
 'adult' 'adults' 'advance' 'advanced' 'advances' 'advantage' 'adventure'
 'advertising' 'advice']

numero_topicos_ngramas = numero_topicos

modelo_lda_ngramas, tabla_topicos_ngramas = entrenar_y_resumir_lda(
    matriz_ngramas,
    vocabulario_ngramas,
    numero_topicos=numero_topicos_ngramas,
    numero_palabras=12
)

tabla_topicos_ngramas

print("Tópicos obtenidos con la representación original basada en unigramas:")
display(tabla_topicos[["topic", "top_words"]])

print("\nTópicos obtenidos con la representación basada en unigramas y bigramas:")
display(tabla_topicos_ngramas[["topic", "top_words"]])

Tópicos obtenidos con la representación original basada en unigramas:

Tópicos obtenidos con la representación basada en unigramas y bigramas:

valores_k_ngramas = [4, 6, 8, 10]

resultados_modelos_ngramas = {}

for k in valores_k_ngramas:
    print("=" * 120)
    print(f"Modelo LDA con unigramas y bigramas, k = {k}")

    modelo_k_ngramas, tabla_k_ngramas = entrenar_y_resumir_lda(
        matriz_ngramas,
        vocabulario_ngramas,
        numero_topicos=k,
        numero_palabras=10
    )

    resultados_modelos_ngramas[k] = {
        "model": modelo_k_ngramas,
        "topics": tabla_k_ngramas
    }

    display(tabla_k_ngramas)

========================================================================================================================
Modelo LDA con unigramas y bigramas, k = 4

========================================================================================================================
Modelo LDA con unigramas y bigramas, k = 6

========================================================================================================================
Modelo LDA con unigramas y bigramas, k = 8

========================================================================================================================
Modelo LDA con unigramas y bigramas, k = 10

resultados_modelos

{4: {'model': LatentDirichletAllocation(max_iter=20, n_components=4, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, donald, president, food, may, know, oba...
  1      1  facebook, star, first, video, world, state, nf...
  2      2  life, world, google, day, food, best, business...
  3      3  week, apple, day, photos, want, top, watch, ip...},
 6: {'model': LatentDirichletAllocation(max_iter=20, n_components=6, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, clinton, better, way, may, know, even, ...
  1      1  star, first, video, state, night, uber, film, ...
  2      2  life, world, things, city, back, take, game, y...
  3      3  apple, week, iphone, travel, think, best, hous...
  4      4  food, photos, day, recipes, best, holiday, wor...
  5      5  trump, health, donald, president, even, good, ...},
 8: {'model': LatentDirichletAllocation(max_iter=20, n_components=8, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, donald, president, obama, clinton, hous...
  1      1  first, star, life, way, uber, data, nfl, olymp...
  2      2  city, things, photos, york, best, study, every...
  3      3  apple, week, iphone, back, help, best, travel,...
  4      4  food, photos, facebook, think, holiday, well, ...
  5      5  health, healthy, may, keep, take, want, good, ...
  6      6  world, day, first, state, little, best, play, ...
  7      7  food, super, day, watch, summer, week, bowl, v...},
 10: {'model': LatentDirichletAllocation(max_iter=20, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, donald, president, live, way, white, fi...
  1      1  first, olympic, sports, star, uber, fans, way,...
  2      2  things, photos, got, world, google, change, ga...
  3      3  travel, apple, nfl, best, think, still, want, ...
  4      4  know, social, want, facebook, job, news, media...
  5      5  health, healthy, take, let, eating, good, keep...
  6      6  day, world, twitter, facebook, first, sleep, c...
  7      7  food, recipes, photos, summer, week, day, reci...
  8      8  women, life, best, world, things, know, photos...
  9      9  trump, apple, clinton, court, week, donald, go...}}

# Uso de spaCy para lematización.

# En Google Colab puede ser necesario descargar el modelo de inglés.
# Si el modelo ya está disponible, esta línea no hace falta volver a ejecutarla.
!python -m spacy download en_core_web_sm

Collecting en-core-web-sm==3.8.0
  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl (12.8 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.8/12.8 MB 51.4 MB/s eta 0:00:00 0:00:01
✔ Download and installation successful
You can now load the package via spacy.load('en_core_web_sm')

import spacy

nlp = spacy.load("en_core_web_sm")

def lematizar_texto(texto):
    """
    Aplica lematización sobre un texto usando spaCy.

    Pasos:
    1. Convierte el texto a minúsculas.
    2. Analiza el texto con el modelo de spaCy.
    3. Conserva únicamente tokens alfabéticos.
    4. Elimina stopwords.
    5. Conserva lemas con longitud mínima de 3 caracteres.

    Devuelve:
    - Una cadena con los lemas separados por espacios.
    """
    documento = nlp(texto.lower())

    lemas = []

    for token in documento:
        if token.is_alpha and not token.is_stop and len(token.lemma_) >= 3:
            lemas.append(token.lemma_)

    return " ".join(lemas)

# Aplicamos la lematización al texto original.
# Este paso puede tardar un poco, dependiendo del número de documentos.

df_reducido["lemmatized_text"] = df_reducido["text"].apply(lematizar_texto)
df_reducido[["text", "processed_text", "lemmatized_text"]].head()

# Es interesante revisar algunas diferencias entre texto preprocesado básico y lematizado
for indice in range(3):
    print("=" * 100)
    print("Categoría:", df_reducido.loc[indice, "category"])

    print("\nTexto original:")
    print(df_reducido.loc[indice, "text"])

    print("\nTexto procesado básico:")
    print(df_reducido.loc[indice, "processed_text"])

    print("\nTexto lematizado:")
    print(df_reducido.loc[indice, "lemmatized_text"])
    print()

====================================================================================================
Categoría: BUSINESS

Texto original:
How to Manage Your Personal Brand. Make no mistake: If you have a Facebook account, an Instagram page, a Twitter profile, you are a brand. Every time you upload a photo, add a link, or post an update, you're putting into the world another idea of yourself and what you stand for.

Texto procesado básico:
manage personal brand mistake facebook account instagram page twitter profile brand every upload photo add link post update putting world another idea stand

Texto lematizado:
manage personal brand mistake facebook account instagram page twitter profile brand time upload photo add link post update put world idea stand

====================================================================================================
Categoría: BUSINESS

Texto original:
It Looks Like Uber's Winning Its War With New York. Grab the popcorn.

Texto procesado básico:
looks uber winning war york grab popcorn

Texto lematizado:
look like uber win war new york grab popcorn

====================================================================================================
Categoría: BUSINESS

Texto original:
The Progressive Promise of Today's Technology. A digital policy for the new century, tailored not just to the moment but for the future, is vital if we are to unleash economic growth, shared prosperity, and the full potential of technology for citizens and consumers. But such a policy architecture requires a new consensus -- on privacy, on security, on customer protections, on growth and mobility.

Texto procesado básico:
progressive promise today technology digital policy century tailored moment future vital unleash economic growth shared prosperity full potential technology citizens consumers policy architecture requires consensus privacy security customer protections growth mobility

Texto lematizado:
progressive promise today technology digital policy new century tailor moment future vital unleash economic growth share prosperity potential technology citizen consumer policy architecture require new consensus privacy security customer protection growth mobility

# Construimos una matriz documento-término con conteos usando el texto lematizado.

vectorizador_lemas = CountVectorizer(
    lowercase=False,
    min_df=5,
    max_df=0.80,
    max_features=3000
)

matriz_lemas = vectorizador_lemas.fit_transform(df_reducido["lemmatized_text"])

vocabulario_lemas = vectorizador_lemas.get_feature_names_out()

print("Dimensiones de la matriz documento-término lematizada:")
print(matriz_lemas.shape)

print("\nTamaño del vocabulario lematizado:", len(vocabulario_lemas))
print("Primeros términos del vocabulario lematizado:")
print(vocabulario_lemas[:50])

Dimensiones de la matriz documento-término lematizada:
(3924, 2402)

Tamaño del vocabulario lematizado: 2402
Primeros términos del vocabulario lematizado:
['abandon' 'abc' 'ability' 'able' 'abortion' 'absolutely' 'abuse'
 'academy' 'accept' 'access' 'accessible' 'accident' 'accord' 'account'
 'accusation' 'accuse' 'achieve' 'acknowledge' 'acquire' 'act' 'action'
 'active' 'activity' 'actor' 'actress' 'actual' 'actually' 'add' 'addict'
 'addition' 'address' 'administration' 'admit' 'adorable' 'adult'
 'advance' 'advanced' 'advantage' 'adventure' 'advertising' 'advice'
 'adviser' 'affair' 'affect' 'afford' 'affordable' 'afraid' 'africa'
 'african' 'aftermath']

valores_k = [4, 6, 8, 10]

resultados_modelos_lemas = {}

for k in valores_k:
    print("=" * 120)
    print(f"Modelo LDA con lematización y {k} tópicos")

    modelo_k_lemas, tabla_k_lemas = entrenar_y_resumir_lda(
        matriz_lemas,
        vocabulario_lemas,
        numero_topicos=k,
        numero_palabras=10
    )

    resultados_modelos_lemas[k] = {
        "model": modelo_k_lemas,
        "topics": tabla_k_lemas
    }

    display(tabla_k_lemas)

========================================================================================================================
Modelo LDA con lematización y 4 tópicos

========================================================================================================================
Modelo LDA con lematización y 6 tópicos

========================================================================================================================
Modelo LDA con lematización y 8 tópicos

========================================================================================================================
Modelo LDA con lematización y 10 tópicos

resultados_modelos_lemas

{4: {'model': LatentDirichletAllocation(max_iter=20, n_components=4, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, want, change, way, say, good, know, bre...
  1      1  new, year, good, travel, time, day, world, hea...
  2      2  day, recipe, photo, video, time, game, olympic...
  3      3  new, trump, food, say, people, report, apple, ...},
 6: {'model': LatentDirichletAllocation(max_iter=20, n_components=6, random_state=42),
  'topics':    topic                                          top_words
  0      0  trump, change, donald, life, way, say, want, p...
  1      1  new, year, time, good, season, life, look, man...
  2      2  day, recipe, apple, week, state, house, video,...
  3      3  food, new, study, people, like, eat, trump, ca...
  4      4  new, photo, travel, good, world, year, know, w...
  5      5  facebook, say, company, health, google, day, y...},
 8: {'model': LatentDirichletAllocation(max_iter=20, n_components=8, random_state=42),
  'topics':    topic                                          top_words
  0      0  want, change, way, time, beer, life, fear, spo...
  1      1  job, new, year, video, look, time, watch, week...
  2      2  week, stress, day, star, life, player, feel, a...
  3      3  food, new, like, study, cancer, win, year, tim...
  4      4  photo, new, travel, good, world, year, hotel, ...
  5      5  facebook, say, google, company, apple, social,...
  6      6  trump, say, state, people, president, donald, ...
  7      7  recipe, day, good, think, health, eat, trump, ...},
 10: {'model': LatentDirichletAllocation(max_iter=20, random_state=42),
  'topics':    topic                                          top_words
  0      0  way, time, life, change, need, beer, want, bre...
  1      1  year, job, season, life, day, time, good, holi...
  2      2  day, week, stress, feel, apple, time, star, li...
  3      3  new, like, amazon, food, cancer, share, apple,...
  4      4  photo, travel, good, new, world, place, hotel,...
  5      5  google, new, say, want, apple, year, help, wor...
  6      6  trump, people, say, donald, state, company, pr...
  7      7  recipe, good, food, think, day, olympic, eat, ...
  8      8  trump, say, clinton, court, obama, president, ...
  9      9  video, new, look, week, game, film, super, wat...}}

# Entrenamos un modelo LDA concreto sobre la versión lematizada para compararlo
# con el modelo principal entrenado sobre processed_text.

numero_topicos_lemas = 8

modelo_lda_lemas = LatentDirichletAllocation(
    n_components=numero_topicos_lemas,
    random_state=42,
    learning_method="batch",
    max_iter=20
)

modelo_lda_lemas.fit(matriz_lemas)

tabla_topicos_lemas = obtener_palabras_topico(
    modelo_lda_lemas,
    vocabulario_lemas,
    numero_palabras=12
)

tabla_topicos_lemas[["topic", "top_words"]]

from sklearn.feature_extraction.text import TfidfVectorizer

vectorizador_tfidf = TfidfVectorizer(
    lowercase=False,
    min_df=5,
    max_df=0.80,
    max_features=3000
)

matriz_tfidf = vectorizador_tfidf.fit_transform(df_reducido["processed_text"])
vocabulario_tfidf = vectorizador_tfidf.get_feature_names_out()

# Las dimensiones son las mismas que para la matriz documento-término. La diferencia
# está en que con TF-IDF cada celda tiene la importancia relativa (peso) de un término
# en un documento dentro del corpus
print("Dimensiones de la matriz TF-IDF:")
print(matriz_tfidf.shape)

Dimensiones de la matriz TF-IDF:
(3924, 2797)

indice_documento = 0

vector_tfidf_documento = matriz_tfidf[indice_documento].toarray()[0]

terminos_tfidf = []

for indice_termino, valor_tfidf in enumerate(vector_tfidf_documento):
    if valor_tfidf > 0:
        terminos_tfidf.append((vocabulario_tfidf[indice_termino], valor_tfidf))

df_tfidf_documento = pd.DataFrame(
    terminos_tfidf,
    columns=["term", "tfidf"]
).sort_values(by="tfidf", ascending=False)

print("Categoría real:", df_reducido.loc[indice_documento, "category"])
print("Texto original:")
print(df_reducido.loc[indice_documento, "text"])
print("\nTérminos con mayor TF-IDF:")
df_tfidf_documento.head(15)

Categoría real: BUSINESS
Texto original:
How to Manage Your Personal Brand. Make no mistake: If you have a Facebook account, an Instagram page, a Twitter profile, you are a brand. Every time you upload a photo, add a link, or post an update, you're putting into the world another idea of yourself and what you stand for.

Términos con mayor TF-IDF:

from sklearn.metrics.pairwise import cosine_similarity

indice_consulta = 0

similitudes = cosine_similarity(
    matriz_tfidf[indice_consulta],
    matriz_tfidf
).flatten()

# Ordenamos documentos por similitud descendente.
# Excluimos el propio documento, que tendrá similitud 1 consigo mismo.
indices_ordenados = similitudes.argsort()[::-1]

documentos_similares = []

for indice in indices_ordenados:
    if indice != indice_consulta:
        documentos_similares.append(indice)
    if len(documentos_similares) == 5:
        break

print("Documento de consulta:")
print("Categoría:", df_reducido.loc[indice_consulta, "category"])
print(df_reducido.loc[indice_consulta, "text"])

print("\nDocumentos más similares según TF-IDF:")
for indice in documentos_similares:
    print("=" * 100)
    print("Similitud:", similitudes[indice])
    print("Categoría:", df_reducido.loc[indice, "category"])
    print(df_reducido.loc[indice, "text"])

Documento de consulta:
Categoría: BUSINESS
How to Manage Your Personal Brand. Make no mistake: If you have a Facebook account, an Instagram page, a Twitter profile, you are a brand. Every time you upload a photo, add a link, or post an update, you're putting into the world another idea of yourself and what you stand for.

Documentos más similares según TF-IDF:
====================================================================================================
Similitud: 0.2693665162274382
Categoría: TECH
Twitter, Instagram Deal Almost Happened For $525 Million: NYT. Relations between Twitter, Instagram and Facebook have soured since Facebook successfully swooped for the photo service. Earlier
====================================================================================================
Similitud: 0.21638978927152627
Categoría: TECH
Facebook Announces Animated Profile Pics. Oh boy.
====================================================================================================
Similitud: 0.19812405766694044
Categoría: TECH
Brace Yourself For Even More Facebook Notifications. Here's the latest update that Facebook hopes will suck you in.
====================================================================================================
Similitud: 0.1613125279881371
Categoría: TECH
How Joining Facebook Is Hurting Instagram. Just months ago Instagram celebrated a billion-dollar buyout. Now it's falling along with Facebook. Read more on The Daily
====================================================================================================
Similitud: 0.16061489365382442
Categoría: POLITICS
'Better Late Than Never': Account Reposting Trump Tweets Verbatim Reacts To Twitter Ban. The @SuspendThePres account reposted Trump's words for months to prove that an average citizen would be banned for such rhetoric.

	link	headline	category	short_description	authors	date
0	https://www.huffpost.com/entry/covid-boosters-...	Over 4 Million Americans Roll Up Sleeves For O...	U.S. NEWS	Health experts said it is too early to predict...	Carla K. Johnson, AP	2022-09-23
1	https://www.huffpost.com/entry/american-airlin...	American Airlines Flyer Charged, Banned For Li...	U.S. NEWS	He was subdued by passengers and crew when he ...	Mary Papenfuss	2022-09-23
2	https://www.huffpost.com/entry/funniest-tweets...	23 Of The Funniest Tweets About Cats And Dogs ...	COMEDY	"Until you have a dog you don't understand wha...	Elyse Wanshel	2022-09-23
3	https://www.huffpost.com/entry/funniest-parent...	The Funniest Tweets From Parents This Week (Se...	PARENTING	"Accidentally put grown-up toothpaste on my to...	Caroline Bologna	2022-09-23
4	https://www.huffpost.com/entry/amy-cooper-lose...	Woman Who Called Cops On Black Bird-Watcher Lo...	U.S. NEWS	Amy Cooper accused investment firm Franklin Te...	Nina Golgowski	2022-09-22

	category	headline	short_description	text	processed_text	topic_0	topic_1	topic_2	topic_3	topic_4	topic_5	topic_6	topic_7	dominant_topic	dominant_topic_weight
0	BUSINESS	How to Manage Your Personal Brand	Make no mistake: If you have a Facebook accoun...	How to Manage Your Personal Brand. Make no mis...	manage personal brand mistake facebook account...	0.005444	0.005445	0.005446	0.005447	0.005449	0.005440	0.961877	0.005453	6	0.961877
1	BUSINESS	It Looks Like Uber's Winning Its War With New ...	Grab the popcorn.	It Looks Like Uber's Winning Its War With New ...	looks uber winning war york grab popcorn	0.017901	0.017900	0.017867	0.017866	0.017861	0.017867	0.017859	0.874879	7	0.874879
2	BUSINESS	The Progressive Promise of Today's Technology	A digital policy for the new century, tailored...	The Progressive Promise of Today's Technology....	progressive promise today technology digital p...	0.005004	0.964974	0.005003	0.005003	0.005004	0.005006	0.005004	0.005002	1	0.964974
3	BUSINESS	Don't Let These 5 Confusing Words Mar Your Image	Tom's an articulate physician, totally able to...	Don't Let These 5 Confusing Words Mar Your Ima...	let confusing words mar image tom articulate p...	0.006957	0.476840	0.006960	0.237759	0.006951	0.250616	0.006966	0.006951	1	0.476840
4	BUSINESS	What You Don't Know About Overnight Success	I've been fighting this thing for 32 years. "O...	What You Don't Know About Overnight Success. I...	know overnight success fighting thing overnigh...	0.007357	0.007358	0.007361	0.007360	0.948491	0.007359	0.007357	0.007358	4	0.948491

	category	headline	short_description	topic_0	dominant_topic_weight
846	ENTERTAINMENT	Love Him or Hate Him: Entrepreneurs Can Learn ...	Entrepreneurs who are just starting out should...	0.970806	0.970806
204	BUSINESS	Maybe The Federal Reserve Has Been Right About...	Hard as it may be for its legion of economic, ...	0.964961	0.964961
3530	WELLNESS	Does Memory Training Really Work?	The latest research suggests memory training d...	0.964952	0.964952
1699	POLITICS	States Must Find Way to Do Right by Home Care ...	When does justice delayed become justice denie...	0.960181	0.960181
3746	WELLNESS	Birth and Death	Lola taught me how to handle adversity and Luc...	0.958258	0.958258

	category	headline	short_description	topic_0	dominant_topic_weight
846	ENTERTAINMENT	Love Him or Hate Him: Entrepreneurs Can Learn ...	Entrepreneurs who are just starting out should...	0.970806	0.970806
204	BUSINESS	Maybe The Federal Reserve Has Been Right About...	Hard as it may be for its legion of economic, ...	0.964961	0.964961
3530	WELLNESS	Does Memory Training Really Work?	The latest research suggests memory training d...	0.964952	0.964952

	category	headline	short_description	topic_1	dominant_topic	dominant_topic_weight
3781	WELLNESS	ACEs: Adverse Childhood Experiences -- and Pro...	Many youth experience events in their early li...	0.967553	1	0.967553
2	BUSINESS	The Progressive Promise of Today's Technology	A digital policy for the new century, tailored...	0.964974	1	0.964974
44	BUSINESS	What Is the Optimal Credit Score?	Truthfully, the highest credit score is not th...	0.964951	1	0.964951

Procesamiento y detección de tópicos con LDA¶

1. Carga del dataset¶

Inspección inicial¶

2. Exploración inicial del corpus¶

2.1. Selección de categorías para la práctica¶

2.2. Reducción del tamaño del corpus¶

3. Construcción del texto de trabajo¶

3.1. Revisión de textos vacíos o demasiado cortos¶

4. Preprocesamiento básico¶

4.1. Comparación entre texto original y texto procesado¶

5. Exploración del texto procesado¶

5.1. Longitud de los documentos procesados¶

6. Representación de la matriz documento-término con conteos¶

6.1. Ejemplo de vector de un documento¶

7. Entrenamiento de un modelo LDA¶

7.1. Palabras principales de cada tópico¶

8. Interpretación de tópicos¶

8.1. Visualización de palabras principales por tópico¶

9. Distribución de tópicos por documento¶

9.1. Documentos representativos por tópico¶

10. Comparación con las categorías reales¶

10.1. Interpretación de la comparación¶

11. Prueba con distintos números de tópicos¶

11.1. Comparación de configuraciones¶

12. Extensión: uso de n-gramas en la representación¶

12.1. Entrenamiento de LDA con unigramas y bigramas¶

12.2. Comparación con la representación basada solo en unigramas¶

12.3. Prueba con distintos números de tópicos usando n-gramas¶

12.4. Conclusión sobre el uso de n-gramas¶

13. Limitaciones observadas¶

14. Extensión opcional: prueba con lematización¶

Conclusión sobre la lematización¶

15. Extensión: representación con TF-IDF¶

15.1. Términos con mayor TF-IDF en un documento¶

15.2. Similitud entre documentos usando TF-IDF¶

16. Conclusiones¶

	word	frequency
0	trump	251
1	day	218
2	world	202
3	best	180
4	photos	176
5	life	162
6	first	156
7	food	152
8	may	146
9	know	143
10	way	137
11	take	129
12	good	128
13	week	125
14	want	122
15	many	119
16	think	114
17	video	114
18	back	112
19	health	112
20	even	111
21	need	111
22	apple	110
23	love	103
24	things	101
25	help	100
26	game	100
27	travel	100
28	every	97
29	last	97

	term	count
3	brand	2
0	account	1
11	page	1
19	update	1
18	twitter	1
17	stand	1
16	putting	1
15	profile	1
14	post	1
13	photo	1
12	personal	1
10	mistake	1
1	add	1
9	manage	1
8	link	1
7	instagram	1
6	idea	1
5	facebook	1
4	every	1
2	another	1

	topic	top_words
0	0	trump, donald, president, obama, clinton, hous...
1	1	first, star, life, way, uber, data, nfl, olymp...
2	2	city, things, photos, york, best, study, every...
3	3	apple, week, iphone, back, help, best, travel,...
4	4	food, photos, facebook, think, holiday, well, ...
5	5	health, healthy, may, keep, take, want, good, ...
6	6	world, day, first, state, little, best, play, ...
7	7	food, super, day, watch, summer, week, bowl, v...

	category	headline	short_description	topic_2	dominant_topic	dominant_topic_weight
3420	TRAVEL	Five Things You Must Buy in the Caribbean - A...	Step away from the two-for-$20 T-shirts, the m...	0.967551	2	0.967551
376	BUSINESS	The Odd Couple In Today's Office: Millennials...	The term 'reverse mentor' was coined and first...	0.967547	2	0.967547
218	BUSINESS	4 Things They Don't Teach You in MBA Programs ...	Having spent six years in two different colleg...	0.964956	2	0.964956

	category	headline	short_description	topic_3	dominant_topic	dominant_topic_weight
715	ENTERTAINMENT	The Second Best Exotic Marigold Hotel: A Movie...	The return viewer to The Best Exotic Marigold ...	0.971728	3	0.971728
3088	TRAVEL	5 Secrets To Having a Smarter -- and Happier -...	Money can destroy your trip, but not necessari...	0.967554	3	0.967554
2460	TECH	App Neutrality Should Be Part of the Net Neutr...	I bet you think the mobile Internet is open. T...	0.966294	3	0.966294

	category	headline	short_description	topic_4	dominant_topic	dominant_topic_weight
265	BUSINESS	Let's Talk Toilet Paper	I once worked in a Fortune 500 paper mill that...	0.972627	4	0.972627
3522	WELLNESS	What Would YOU Do With A Second Chance In Life?	On Dec. 26, 2004, my life changed forever. Tha...	0.967530	4	0.967530
3830	WELLNESS	Meditation Tips: Does Meditation Have to Be Se...	Some people think that their thoughts and feel...	0.964964	4	0.964964

	category	headline	short_description	topic_5	dominant_topic	dominant_topic_weight
1497	POLITICS	Reflections on Justice After Ferguson, MO: Wh...	It turns out that justice, like so much of wha...	0.974966	5	0.974966
3769	WELLNESS	Better Data Will Lead to Better Care for Those...	People living with multiple chronic conditions...	0.969798	5	0.969798
2153	SPORTS	NFL-Backed Youth Program Says It Reduced Concu...	As increasing numbers of parents keep their ch...	0.963505	5	0.963505

	category	headline	short_description	topic_6	dominant_topic	dominant_topic_weight
775	ENTERTAINMENT	Filmmaker Fights the Stigma Against Mental Ill...	The best thing about the #MyMentalHealthStory ...	0.971730	6	0.971730
21	BUSINESS	How Clean Energy Works for Colorado	Smart clean energy policies like Colorado's pe...	0.966309	6	0.966309
3872	WELLNESS	Can We Predict How Stressed Out You Are From T...	We set up a tracking poll in February to gauge...	0.966305	6	0.966305

	category	headline	short_description	topic_7	dominant_topic	dominant_topic_weight
1401	FOOD & DRINK	9 Labor Day Grill Favorites	For most children, Labor Day is a bittersweet ...	0.973444	7	0.973444
1408	FOOD & DRINK	Labor Day Cocktails	Labor Day weekend is summer's last long weeken...	0.967566	7	0.967566
3039	TRAVEL	Chefs A-Twitter In London, New York And Paris:...	When planning a trip, I am prejudiced in favor...	0.967545	7	0.967545

	topic	top_words
0	0	trump, donald, president, food, may, know, oba...
1	1	facebook, star, first, video, world, state, nf...
2	2	life, world, google, day, food, best, business...
3	3	week, apple, day, photos, want, top, watch, ip...

	term	tfidf
3	brand	0.436596
8	link	0.238437
10	mistake	0.234093
9	manage	0.234093
15	profile	0.230261
0	account	0.230261
11	page	0.223733
19	update	0.218298
16	putting	0.213643
1	add	0.207711
6	idea	0.198352
17	stand	0.197024
7	instagram	0.195748
14	post	0.195748
12	personal	0.191093

	topic	top_words
0	0	trump, clinton, better, way, may, know, even, ...
1	1	star, first, video, state, night, uber, film, ...
2	2	life, world, things, city, back, take, game, y...
3	3	apple, week, iphone, travel, think, best, hous...
4	4	food, photos, day, recipes, best, holiday, wor...
5	5	trump, health, donald, president, even, good, ...

	topic	top_words
0	0	trump, donald, president, live, way, white, fi...
1	1	first, olympic, sports, star, uber, fans, way,...
2	2	things, photos, got, world, google, change, ga...
3	3	travel, apple, nfl, best, think, still, want, ...
4	4	know, social, want, facebook, job, news, media...
5	5	health, healthy, take, let, eating, good, keep...
6	6	day, world, twitter, facebook, first, sleep, c...
7	7	food, recipes, photos, summer, week, day, reci...
8	8	women, life, best, world, things, know, photos...
9	9	trump, apple, clinton, court, week, donald, go...

	topic	top_words
0	0	day, travel, free, around, best, photos, busin...
1	1	day, video, game, women, football, team, nfl, ...
2	2	recipes, photos, season, holiday, need, job, u...
3	3	trump, donald, donald trump, health, president...
4	4	food, world, better, twitter, may, life, reall...
5	5	apple, week, food, life, look, back, watch, ta...
6	6	best, want, know, love, find, travel, right, s...
7	7	facebook, social, first, want, olympic, media,...

	topic	top_words
0	0	day, best, know, free, top, recipe, facebook, ...
1	1	day, first, world, game, team, women, video, w...
2	2	food, want, season, life, change, ways, think,...
3	3	trump, donald, apple, donald trump, president,...

	topic	top_words
0	0	facebook, day, google, recipe, health, busines...
1	1	day, social, super, world, video, first, women...
2	2	recipes, want, photos, change, season, life, u...
3	3	trump, donald, donald trump, president, house,...
4	4	photos, world, best, food, city, travel, may, ...
5	5	apple, week, love, life, know, look, food, tak...

	topic	top_words
0	0	day, health, james, old, data, fun, watch, bus...
1	1	women, game, super, team, bowl, first, way, su...
2	2	change, life, american, stop, san, video, goog...
3	3	trump, donald, donald trump, president, clinto...
4	4	world, travel, twitter, city, may, best, reall...
5	5	apple, iphone, week, life, know, good, rumors,...
6	6	facebook, state, right, home, company, best, s...
7	7	first, want, help, social, world, media, show,...
8	8	food, day, photos, recipes, recipe, best, love...
9	9	watch, week, see, videos, know, movie, youtube...

	topic	top_words
0	0	trump, want, change, way, say, good, know, bre...
1	1	new, year, good, travel, time, day, world, hea...
2	2	day, recipe, photo, video, time, game, olympic...
3	3	new, trump, food, say, people, report, apple, ...

	topic	top_words
0	0	trump, change, donald, life, way, say, want, p...
1	1	new, year, time, good, season, life, look, man...
2	2	day, recipe, apple, week, state, house, video,...
3	3	food, new, study, people, like, eat, trump, ca...
4	4	new, photo, travel, good, world, year, know, w...
5	5	facebook, say, company, health, google, day, y...

	topic	top_words
0	0	want, change, way, time, beer, life, fear, spo...
1	1	job, new, year, video, look, time, watch, week...
2	2	week, stress, day, star, life, player, feel, a...
3	3	food, new, like, study, cancer, win, year, tim...
4	4	photo, new, travel, good, world, year, hotel, ...
5	5	facebook, say, google, company, apple, social,...
6	6	trump, say, state, people, president, donald, ...
7	7	recipe, day, good, think, health, eat, trump, ...

	topic	top_words
0	0	way, time, life, change, need, beer, want, bre...
1	1	year, job, season, life, day, time, good, holi...
2	2	day, week, stress, feel, apple, time, star, li...
3	3	new, like, amazon, food, cancer, share, apple,...
4	4	photo, travel, good, new, world, place, hotel,...
5	5	google, new, say, want, apple, year, help, wor...
6	6	trump, people, say, donald, state, company, pr...
7	7	recipe, good, food, think, day, olympic, eat, ...
8	8	trump, say, clinton, court, obama, president, ...
9	9	video, new, look, week, game, film, super, wat...