Exploratory Data Analysis (EDA) is nice and easy, if the entities you look at are of one of the typical levels of measurement (see below). But if one feature is a set, it becomes harder. In this post, I want to show a couple of possibilities.
Level of measurement ¶
If you have features of this level of measurement.
| Scale | |||||
|---|---|---|---|---|---|
| Qualitative (categorical) | Quantitative (metric) | ||||
| Nominal- | Ordinal- | Intervall- | Ratio- | Absolute | |
| Empirical relations | Equivalence | Equivalence order |
Equivalence order emp. addition |
Equivalence order emp. addition emp. multipliation |
Equivalence order emp. addition emp. multipliation |
| Allowed transformationen | m' = f(m) f bijektive |
m' = f(m) f strictly monotonous |
m' = am+b with a>0 |
m' = am with a>0 |
m' = m |
| Examples of this scale | Telephone numbers, license plates, types, postal codes, gender | Grades, Degrees of hardness, wind force | Temp. in °C, °F, calendar time, geographic height | Mass, length, el. current | Number of particles, number of errors |
| Values of m | numbers, names, symbols | usually natural numbers | usually real numbers | usually real numbers > 0 | usually natural numbers |
Datasets ¶
dblp ¶
dblp is a bibliography website which contains publication data from almost 2 million publications.
Use DBLPParser to create a CSV file.
from collections import Counter
import numpy as np
import pandas as pd
import progressbar
import networkx as nx
from itertools import combinations
import clana.io
import clana.visualize_cm
# Load the data
df = pd.read_csv("articles.csv")
df["author"] = df["author"].str.split("::")
# Analyze the data
df = df[~df["author"].isna()]
authors = [author for authorset in df["author"].tolist() for author in authorset]
author_count = Counter(authors)
print("* Publications: {}".format(len(df)))
print("* Unique elements: {}".format(len(author_count)))
print("* Most common:")
most_common = sorted(author_count.items(), key=lambda n: n[1], reverse=True)
for name, count in most_common[:10]:
print(" {:>4}x {}".format(count, name))
unique_authors = sorted(list(author_count.keys()))
def get_biggest_clusters(edges, n=10):
G = nx.Graph()
for authorset in edges.tolist():
for author in authorset:
G.add_node(author)
for authorset in progressbar.progressbar(df["author"].tolist()[:10_000]):
for author1, author2 in combinations(authorset, 2):
G.add_edge(author1, author2)
print("Edges were added")
components = [c for c in sorted(nx.connected_components(G), key=len, reverse=True)]
return components[:n]
def create_matrix(nodes, edges):
n2i = dict([(node, i) for i, node in enumerate(sorted(nodes))])
# node to index
mat = np.zeros((len(nodes), len(nodes)), dtype=np.int32)
for edge in edges:
for a, b in combinations(edge, 2):
if a not in n2i or b not in n2i:
continue
mat[n2i[a]][n2i[b]] += 1
if a != b:
mat[n2i[b]][n2i[a]] += 1
return mat, sorted(nodes)
components = get_biggest_clusters(df["author"])
print("* Biggest clusters: {}".format([len(el) for el in components]))
component_w_publications = [(author, author_count[author]) for author in components[0]]
component_w_publications = sorted(
component_w_publications, key=lambda n: n[1], reverse=True
)
authors = [author for author, count in component_w_publications[:1_00]]
mat, labels = create_matrix(authors, df["author"].tolist())
clana.visualize_cm.main(
"coauthors.json",
perm_file="",
steps=1_000_000,
labels_file="labels.json",
zero_diagonal=False,
output="cm-ordered.pdf",
)
clana.io.write_cm("coauthors.json", mat)
clana.io.write_labels("labels.json", labels)
Results:
* Publications: 2,054,474
* Unique elements: 1,475,717
* Most common
1181x H. Vincent Poor
789x Lajos Hanzo
767x Witold Pedrycz
747x Mohamed-Slim Alouini
615x Chin-Chen Chang 0001
607x Dacheng Tao
591x Victor C. M. Leung
570x Wei Zhang
562x Wei Li
554x Wei Wang
* Biggest clusters (under first 10^6 publications): [761987, 52, 45, 44, 32, 31, 29, 28, 28, 28]
Then you can apply confusion matrix ordering to find authors who often work together (click on it to see large version):
The CMO technique is described in
Thoma, Martin. "Analysis and optimization of convolutional neural network architectures." arXiv preprint arXiv:1707.09725 (2017). Chapter 5.2.
MovieLens 20M ¶
from collections import Counter
import numpy as np
import pandas as pd
import progressbar
import networkx as nx
from itertools import combinations
import clana.io
import clana.visualize_cm
# Load the data
df = pd.read_csv("movies.csv")
df["genres"] = df["genres"].str.split("|")
# Analyze the data
list_values = [value for valueset in df["genres"].tolist() for value in valueset]
value_count = Counter(list_values)
print("* Movies: {}".format(len(df)))
print("* Unique genres: {}".format(len(value_count)))
print("* Most common:")
most_common = sorted(value_count.items(), key=lambda n: n[1], reverse=True)
for name, count in most_common[:10]:
print(" {:>4}x {}".format(count, name))
unique_genres = sorted(list(value_count.keys()))
def get_biggest_clusters(edges, n=10):
G = nx.Graph()
for authorset in edges.tolist():
for author in authorset:
G.add_node(author)
for authorset in progressbar.progressbar(df["genres"].tolist()[:10_000]):
for author1, author2 in combinations(authorset, 2):
G.add_edge(author1, author2)
print("Edges were added")
components = [c for c in sorted(nx.connected_components(G), key=len, reverse=True)]
return components[:n]
def create_matrix(nodes, edges):
n2i = dict([(node, i) for i, node in enumerate(sorted(nodes))])
# node to index
mat = np.zeros((len(nodes), len(nodes)), dtype=np.int32)
for edge in edges:
for a, b in combinations(edge, 2):
if a not in n2i or b not in n2i:
continue
mat[n2i[a]][n2i[b]] += 1
if a != b:
mat[n2i[b]][n2i[a]] += 1
return mat, sorted(nodes)
components = get_biggest_clusters(df["genres"])
print("* Biggest clusters: {}".format([len(el) for el in components]))
component_w_publications = [(author, value_count[author]) for author in components[0]]
component_w_publications = sorted(
component_w_publications, key=lambda n: n[1], reverse=True
)
authors = [author for author, count in component_w_publications[:1_00]]
mat, labels = create_matrix(authors, df["genres"].tolist())
clana.io.write_cm("genre-combinations.json", mat)
clana.io.write_labels("labels.json", labels)
clana.visualize_cm.main(
"genre-combinations.json",
perm_file="",
steps=1_000_000,
labels_file="labels.json",
zero_diagonal=False,
output="cm-genre-combinations.pdf",
)
Results:
* Movies: 27278
* Unique genres: 20
* Most common:
13344x Drama
8374x Comedy
4178x Thriller
4127x Romance
3520x Action
2939x Crime
2611x Horror
2471x Documentary
2329x Adventure
1743x Sci-Fi
CMO:
