Merge pull request #117 from pi-hole/feature/shm-API-update

Update dashboard components to use new shared memory API data

make-fake-data.js

@@ -8,17 +8,16 @@
 *  This file is copyright under the latest version of the EUPL.
 *  Please see LICENSE file for your rights under this license. */
 
-const faker = require('faker');
-const fs = require('fs-extra');
+const faker = require("faker");
+const fs = require("fs-extra");
 
 function unique(generator, size) {
   const data = [];
 
-  while(data.length !== size) {
+  while (data.length !== size) {
     const item = generator();
 
-    if(!data.includes(item))
-      data.push(item);
+    if (!data.includes(item)) data.push(item);
   }
 
   return data;
@@ -43,82 +42,111 @@ function pastDate() {
 function history(length) {
   const startDate = pastDate();
 
-  return (new Array(length)).fill(null).map((_, i) => {
+  return new Array(length).fill(null).map((_, i) => {
     const isIPv4 = faker.random.boolean();
     const isHostname = faker.random.boolean();
     return [
       startDate + i,
       isIPv4 ? "IPv4" : "IPv6",
       faker.internet.domainName(),
-      isHostname ? faker.internet.domainWord() + ".local" : isIPv4 ? faker.internet.ip() : faker.internet.ipv6(),
+      isHostname
+        ? faker.internet.domainWord() + ".local"
+        : isIPv4
+          ? faker.internet.ip()
+          : faker.internet.ipv6(),
       Math.floor(Math.random() * 5) + 1
     ];
   });
 }
 
 function summary() {
-  const total = faker.random.number();
-  const ads = faker.random.number({max: total});
+  const queryTypeTotals = [
+    faker.random.number(), // A
+    faker.random.number(), // AAAA
+    faker.random.number(), // ANY
+    faker.random.number(), // SRV
+    faker.random.number(), // SOA
+    faker.random.number(), // PTR
+    faker.random.number() // TXT
+  ];
+
+  const total = queryTypeTotals.reduce(
+    (previous, current) => previous + current,
+    0
+  );
+  const blocked = faker.random.number({ max: total });
   const forwarded = faker.random.number({ max: total });
   const clients = faker.random.number();
 
   return {
-    "domains_blocked": faker.random.number(),
-    "total_queries": total,
-    "blocked_queries": ads,
-    "percent_blocked": ads * 100 / total,
-    "unique_domains": faker.random.number({ max: total }),
-    "forwarded_queries": forwarded,
-    "cached_queries": total - forwarded,
-    "total_clients": clients,
-    "unique_clients": faker.random.number({ max: clients }),
-    ... status()
+    gravity_size: faker.random.number(),
+    total_queries: {
+      A: queryTypeTotals[0],
+      AAAA: queryTypeTotals[1],
+      ANY: queryTypeTotals[2],
+      SRV: queryTypeTotals[3],
+      SOA: queryTypeTotals[4],
+      PTR: queryTypeTotals[5],
+      TXT: queryTypeTotals[6]
+    },
+    blocked_queries: blocked,
+    percent_blocked: (blocked * 100) / total,
+    unique_domains: faker.random.number({ max: total }),
+    forwarded_queries: forwarded,
+    cached_queries: total - forwarded,
+    reply_types: {
+      IP: total / 5,
+      CNAME: total / 5,
+      DOMAIN: total / 5,
+      NODATA: total / 5,
+      NXDOMAIN: total / 5
+    },
+    total_clients: clients,
+    active_clients: clients - faker.random.number({ max: clients }),
+    ...status()
   };
 }
 
 function queryTypes() {
   const data = [];
   const total = faker.random.number();
-  const names = [
-    "A (IPv4)",
-    "AAAA (IPv6)",
-    "ANY",
-    "SRV",
-    "SOA",
-    "PTR",
-    "TXT"
-  ];
-  const step = total / names.length;
+  const names = ["A", "AAAA", "ANY", "SRV", "SOA", "PTR", "TXT"];
+  const count = total / names.length;
 
-  for(const name of names) {
-    data.push({
-      name,
-      percent: step / total
-    });
+  for (const name of names) {
+    data.push({ name, count });
   }
 
   return data;
 }
 
-function forwardDestinations(length) {
-  const totalQueries = faker.random.number();
-  const destinations = [];
+function upstreams(length) {
+  const upstreams = [];
   const numbers = [];
   const names = unique(faker.internet.domainName, length);
   const ipAddrs = unique(faker.internet.ip, length);
+  let forwardedQueries = 0;
 
-  for(let i = 0; i < length; i++)
-    numbers.push(faker.random.number({ max: totalQueries }));
+  for (let i = 0; i < length; i++) {
+    const number = faker.random.number();
 
-  for(let i = 0; i < length; i++) {
-    destinations.push({
+    forwardedQueries += number;
+    numbers.push(number);
+  }
+
+  for (let i = 0; i < length; i++) {
+    upstreams.push({
       name: names[i],
       ip: ipAddrs[i],
-      percent: numbers[i] / totalQueries
+      count: numbers[i]
     });
   }
 
-  return destinations;
+  return {
+    upstreams,
+    forwarded_queries: forwardedQueries,
+    total_queries: forwardedQueries + faker.random.number()
+  };
 }
 
 function topList(length, max, fakeData) {
@@ -126,12 +154,12 @@ function topList(length, max, fakeData) {
   const numbers = [];
   const domains = unique(fakeData, length);
 
-  for(let i = 0; i < length; i++)
+  for (let i = 0; i < length; i++)
     numbers.push(faker.random.number({ max: max }));
 
-  numbers.sort((a, b) => parseInt(a, 10) > parseInt(b, 10) ? -1 : 1);
+  numbers.sort((a, b) => (parseInt(a, 10) > parseInt(b, 10) ? -1 : 1));
 
-  for(let i = 0; i < length; i++) {
+  for (let i = 0; i < length; i++) {
     result.push({
       domain: domains[i],
       count: numbers[i]
@@ -145,8 +173,8 @@ function topBlocked(length) {
   const totalQueries = faker.random.number();
 
   return {
-    "top_blocked": topList(length, totalQueries, faker.internet.domainName),
-    "blocked_queries": totalQueries
+    top_domains: topList(length, totalQueries, faker.internet.domainName),
+    blocked_queries: totalQueries
   };
 }
 
@@ -154,8 +182,8 @@ function topDomains(length) {
   const totalQueries = faker.random.number();
 
   return {
-    "top_domains": topList(length, totalQueries, faker.internet.domainName),
-    "total_queries": totalQueries
+    top_domains: topList(length, totalQueries, faker.internet.domainName),
+    total_queries: totalQueries
   };
 }
 
@@ -165,12 +193,12 @@ function topClients(length) {
   const numbers = [];
   const clients = unique(faker.internet.ip, length);
 
-  for(let i = 0; i < length; i++)
+  for (let i = 0; i < length; i++)
     numbers.push(faker.random.number({ max: totalQueries }));
 
-  numbers.sort((a, b) => parseInt(a, 10) > parseInt(b, 10) ? -1 : 1);
+  numbers.sort((a, b) => (parseInt(a, 10) > parseInt(b, 10) ? -1 : 1));
 
-  for(let i = 0; i < length; i++) {
+  for (let i = 0; i < length; i++) {
     top_clients.push({
       name: "",
       ip: clients[i],
@@ -186,47 +214,42 @@ function topClients(length) {
 
 function clientsOverTime(range, size) {
   const startDate = pastDate();
-  const clients = unique(faker.internet.ip, size)
-    .map(ip => ({ name: faker.internet.domainWord(), ip }));
+  const clients = unique(faker.internet.ip, size).map(ip => ({
+    name: faker.internet.domainWord(),
+    ip
+  }));
   const graph = [];
 
-  for(let i = 0; i < range; i++) {
+  for (let i = 0; i < range; i++) {
     graph.push({
       timestamp: startDate + 600 * i,
-      data: (new Array(size)).fill(null).map(() => faker.random.number())
+      data: new Array(size).fill(null).map(() => faker.random.number())
     });
   }
 
   return {
-    "over_time": graph,
-    "clients": clients
+    over_time: graph,
+    clients: clients
   };
 }
 
 function historyOverTime(range) {
   const startDate = pastDate();
-  const domains = [];
-  const ads = [];
+  const history = [];
 
-  for(let i = 0; i < range; i++) {
-    const numDomains = faker.random.number();
-    const numAds = faker.random.number({ max: numDomains });
-    const time = startDate + 600 * i;
+  for (let i = 0; i < range; i++) {
+    const totalQueries = faker.random.number();
+    const blockedQueries = faker.random.number({ max: totalQueries });
+    const timestamp = startDate + 600 * i;
 
-    domains.push({
-      timestamp: time,
-      count: numDomains
-    });
-    ads.push({
-      timestamp: time,
-      count: numAds
+    history.push({
+      timestamp,
+      total_queries: totalQueries + blockedQueries,
+      blocked_queries: blockedQueries
     });
   }
 
-  return {
-    "domains_over_time": domains,
-    "blocked_over_time": ads
-  };
+  return history;
 }
 
 function write(filePath, data) {
@@ -235,71 +258,108 @@ function write(filePath, data) {
 
 function getNetworkInfo() {
   return {
-    "interface": faker.random.arrayElement(["eth0", "eth1", "wlan0", "wlan1"]),
-    "ipv4_address": faker.internet.ip(),
-    "ipv6_address": faker.internet.ipv6(),
-    "hostname": faker.random.word().toLowerCase().split(" ",2)[0]
+    interface: faker.random.arrayElement(["eth0", "eth1", "wlan0", "wlan1"]),
+    ipv4_address: faker.internet.ip(),
+    ipv6_address: faker.internet.ipv6(),
+    hostname: faker.random
+      .word()
+      .toLowerCase()
+      .split(" ", 2)[0]
   };
 }
 
 function getFTLdb() {
   return {
-    "queries": faker.random.number({min:50000, max:1000000}),
-    "filesize": faker.random.number({min:1000, max:100000}),
-    "sqlite_version": "3.0.1"
+    queries: faker.random.number({ min: 50000, max: 1000000 }),
+    filesize: faker.random.number({ min: 1000, max: 100000 }),
+    sqlite_version: "3.0.1"
   };
 }
 
 function getVersionInfo() {
   return {
-    "api": {
-      "branch": faker.random.arrayElement(["master", "development", "FTL", "beta", "test"]),
-      "hash": faker.internet.color().substring(1) + faker.random.number(9),
-      "tag": "vDev"
+    api: {
+      branch: faker.random.arrayElement([
+        "master",
+        "development",
+        "FTL",
+        "beta",
+        "test"
+      ]),
+      hash: faker.internet.color().substring(1) + faker.random.number(9),
+      tag: "vDev"
     },
-    "core": {
-      "branch": faker.random.arrayElement(["master", "development", "FTL", "beta", "test"]),
-      "hash": faker.internet.color().substring(1) + faker.random.number(9),
-      "tag": "vDev"
+    core: {
+      branch: faker.random.arrayElement([
+        "master",
+        "development",
+        "FTL",
+        "beta",
+        "test"
+      ]),
+      hash: faker.internet.color().substring(1) + faker.random.number(9),
+      tag: "vDev"
     },
-    "ftl": {
-      "branch": faker.random.arrayElement(["master", "development", "FTL", "beta", "test"]),
-      "hash": faker.internet.color().substring(1) + faker.random.number(9),
-      "tag": "vDev"
+    ftl: {
+      branch: faker.random.arrayElement([
+        "master",
+        "development",
+        "FTL",
+        "beta",
+        "test"
+      ]),
+      hash: faker.internet.color().substring(1) + faker.random.number(9),
+      tag: "vDev"
     },
-    "web": {
-      "branch": faker.random.arrayElement(["master", "development", "FTL", "beta", "test"]),
-      "hash": faker.internet.color().substring(1) + faker.random.number(9),
-      "tag": "vDev"
+    web: {
+      branch: faker.random.arrayElement([
+        "master",
+        "development",
+        "FTL",
+        "beta",
+        "test"
+      ]),
+      hash: faker.internet.color().substring(1) + faker.random.number(9),
+      tag: "vDev"
     }
   };
 }
 
 function getDHCPInfo() {
   return {
-    "active": faker.random.boolean(),
-    "ip_start": faker.internet.ip(),
-    "ip_end": faker.internet.ip(),
-    "router_ip": faker.internet.ip(),
-    "lease_time": faker.random.number({min:1, max:99}),
-    "domain": faker.random.word().toLowerCase().split(" ",2)[0],
-    "ipv6_support": faker.random.boolean()
+    active: faker.random.boolean(),
+    ip_start: faker.internet.ip(),
+    ip_end: faker.internet.ip(),
+    router_ip: faker.internet.ip(),
+    lease_time: faker.random.number({ min: 1, max: 99 }),
+    domain: faker.random
+      .word()
+      .toLowerCase()
+      .split(" ", 2)[0],
+    ipv6_support: faker.random.boolean()
   };
 }
 
 function getDNSInfo() {
   return {
-    "upstream_dns": [ faker.internet.ip(), faker.internet.ip(), faker.internet.ip() ],
-    "options": {
-      "fqdn_required": faker.random.boolean(),
-      "bogus_priv": faker.random.boolean(),
-      "dnssec": faker.random.boolean(),
-      "listening_type": faker.random.arrayElement(["single", "lan", "all"]),
+    upstream_dns: [
+      faker.internet.ip(),
+      faker.internet.ip(),
+      faker.internet.ip()
+    ],
+    options: {
+      fqdn_required: faker.random.boolean(),
+      bogus_priv: faker.random.boolean(),
+      dnssec: faker.random.boolean(),
+      listening_type: faker.random.arrayElement(["single", "lan", "all"])
     },
-    "conditional_forwarding": {
-      "enabled": faker.random.boolean(),
-      "router_ip": faker.internet.ip(),
-      "domain": faker.random.word().toLowerCase().split(" ",2)[0]
+    conditional_forwarding: {
+      enabled: faker.random.boolean(),
+      router_ip: faker.internet.ip(),
+      domain: faker.random
+        .word()
+        .toLowerCase()
+        .split(" ", 2)[0]
     }
   };
 }
@@ -325,7 +385,7 @@ write("public/fakeAPI/stats/overTime/clients", clientsOverTime(144, 5));
 write("public/fakeAPI/stats/summary", summary());
 write("public/fakeAPI/stats/history", history(5000));
 write("public/fakeAPI/stats/query_types", queryTypes());
-write("public/fakeAPI/stats/forward_destinations", forwardDestinations(3));
+write("public/fakeAPI/stats/upstreams", upstreams(3));
 write("public/fakeAPI/stats/top_blocked", topBlocked(10));
 write("public/fakeAPI/stats/top_domains", topDomains(10));
 write("public/fakeAPI/stats/top_clients", topClients(10));

src/components/dashboard/GenericDoughnutChart.js

@@ -184,14 +184,18 @@ export const loadingProps = {
   labels: []
 };
 
-export default ({ apiCall, title, ...props }) => (
+export default ({ apiCall, title, apiHandler, ...props }) => (
   <WithAPIData
     apiCall={apiCall}
     renderInitial={() => (
       <GenericDoughnutChart title={title} {...loadingProps} {...props} />
     )}
     renderOk={data => (
-      <GenericDoughnutChart title={title} {...transformData(data)} {...props} />
+      <GenericDoughnutChart
+        title={title}
+        {...transformData(apiHandler(data))}
+        {...props}
+      />
     )}
     renderErr={() => (
       <GenericDoughnutChart title={title} {...loadingProps} {...props} />

src/components/dashboard/QueriesGraph.js

@@ -159,12 +159,11 @@ class QueriesGraph extends Component {
  */
 export const transformData = data => {
   // Remove last data point as it's not yet finished
-  const blockedData = data.blocked_over_time.slice(0, -1);
-  const allowedData = data.domains_over_time.slice(0, -1);
+  data = data.slice(0, -1);
 
-  const labels = allowedData.map(step => new Date(1000 * step.timestamp));
-  const domains_over_time = allowedData.map(step => step.count);
-  const blocked_over_time = blockedData.map(step => step.count);
+  const labels = data.map(step => new Date(1000 * step.timestamp));
+  const domains_over_time = data.map(step => step.total_queries);
+  const blocked_over_time = data.map(step => step.blocked_queries);
 
   return {
     loading: false,

src/components/dashboard/QueryTypesChart.js

@@ -14,7 +14,21 @@ import api from "../../util/api";
 import GenericDoughnutChart from "./GenericDoughnutChart";
 
 const QueryTypesChart = ({ t }) => (
-  <GenericDoughnutChart title={t("Query Types")} apiCall={api.getQueryTypes} />
+  <GenericDoughnutChart
+    title={t("Query Types")}
+    apiCall={api.getQueryTypes}
+    apiHandler={data => {
+      const total = data.reduce(
+        (previous, current) => previous + current.count,
+        0
+      );
+
+      return data.map(item => ({
+        name: item.name,
+        percent: total > 0 ? (item.count * 100) / total : 0
+      }));
+    }}
+  />
 );
 
 export default translate("dashboard")(QueryTypesChart);

src/components/dashboard/SummaryStats.js

@@ -97,11 +97,13 @@ class SummaryStats extends Component {
  * @returns {*} the transformed props
  */
 export const transformData = data => ({
-  totalQueries: data.total_queries.toLocaleString(),
+  totalQueries: Object.keys(data.total_queries)
+    .reduce((total, queryType) => total + data.total_queries[queryType], 0)
+    .toLocaleString(),
   blockedQueries: data.blocked_queries.toLocaleString(),
   percentBlocked: data.percent_blocked.toFixed(2).toLocaleString() + "%",
-  gravityDomains: data.domains_blocked.toLocaleString(),
-  uniqueClients: data.unique_clients
+  gravityDomains: data.gravity_size.toLocaleString(),
+  uniqueClients: data.total_clients
 });
 
 /**

src/components/dashboard/SummaryStats.test.js

@@ -17,25 +17,40 @@ import {
 } from "./SummaryStats";
 
 const fakeData = {
-  domains_blocked: 10193,
-  total_queries: 58787,
-  blocked_queries: 30175,
-  percent_blocked: 51.329375542211714,
-  unique_domains: 33209,
-  forwarded_queries: 33149,
-  cached_queries: 25638,
-  total_clients: 76083,
-  unique_clients: 28949,
-  status: "enabled"
+  active_clients: 2,
+  blocked_queries: 2281,
+  cached_queries: 3573,
+  forwarded_queries: 19154,
+  gravity_size: 129538,
+  percent_blocked: 10.0,
+  reply_types: {
+    CNAME: 0,
+    DOMAIN: 2,
+    IP: 1300,
+    NODATA: 564,
+    NXDOMAIN: 12
+  },
+  status: "enabled",
+  total_clients: 2,
+  total_queries: {
+    A: 14802,
+    AAAA: 7754,
+    ANY: 0,
+    PTR: 29,
+    SOA: 0,
+    SRV: 228,
+    TXT: 0
+  },
+  unique_domains: 15
 };
 
 it("transforms the API data correctly", () => {
   const expectedProps = {
-    totalQueries: "58,787",
-    blockedQueries: "30,175",
-    percentBlocked: "51.33%",
-    gravityDomains: "10,193",
-    uniqueClients: 28949
+    totalQueries: "22,813",
+    blockedQueries: "2,281",
+    percentBlocked: "10.00%",
+    gravityDomains: "129,538",
+    uniqueClients: 2
   };
 
   expect(transformData(fakeData)).toEqual(expectedProps);
@@ -49,18 +64,10 @@ it("displays summary stats correctly", async () => {
   await tick();
   wrapper.update();
 
-  expect(wrapper.childAt(0).find("h3")).toHaveText(
-    fakeData.total_queries.toLocaleString()
-  );
-  expect(wrapper.childAt(1).find("h3")).toHaveText(
-    fakeData.blocked_queries.toLocaleString()
-  );
-  expect(wrapper.childAt(2).find("h3")).toHaveText(
-    fakeData.percent_blocked.toFixed(2).toLocaleString() + "%"
-  );
-  expect(wrapper.childAt(3).find("h3")).toHaveText(
-    fakeData.domains_blocked.toLocaleString()
-  );
+  expect(wrapper.childAt(0).find("h3")).toHaveText("22,813");
+  expect(wrapper.childAt(1).find("h3")).toHaveText("2,281");
+  expect(wrapper.childAt(2).find("h3")).toHaveText("10.00%");
+  expect(wrapper.childAt(3).find("h3")).toHaveText("129,538");
 });
 
 it("displays error message correctly", async () => {

src/components/dashboard/TopBlocked.js

@@ -21,7 +21,7 @@ import TopTable from "./TopTable";
  */
 export const transformData = data => ({
   totalBlocked: data.blocked_queries,
-  topBlocked: data.top_blocked
+  topBlocked: data.top_domains
 });
 
 /**

src/components/dashboard/TopBlocked.test.js

@@ -12,7 +12,7 @@ import React from "react";
 import { transformData, generateRows } from "./TopBlocked";
 
 const fakeData = {
-  top_blocked: [
+  top_domains: [
     { domain: "jaron.info", count: 54316 },
     { domain: "candelario.info", count: 47470 },
     { domain: "delphia.info", count: 41629 },
@@ -31,11 +31,11 @@ it("transforms the API data correctly", async () => {
   const data = transformData(fakeData);
 
   expect(data.totalBlocked).toEqual(fakeData.blocked_queries);
-  expect(data.topBlocked).toEqual(fakeData.top_blocked);
+  expect(data.topBlocked).toEqual(fakeData.top_domains);
 });
 
 it("creates an appropriately sized table", async () => {
   const rows = generateRows(key => key)(transformData(fakeData));
 
-  expect(rows).toHaveLength(fakeData.top_blocked.length);
+  expect(rows).toHaveLength(fakeData.top_domains.length);
 });

src/components/dashboard/UpstreamsChart.js

@@ -3,7 +3,7 @@
 *  Network-wide ad blocking via your own hardware.
 *
 *  Web Interface
-*  Forward Destinations Chart component
+*  Upstream Servers / Forward Destinations Chart component
 *
 *  This file is copyright under the latest version of the EUPL.
 *  Please see LICENSE file for your rights under this license. */
@@ -13,11 +13,18 @@ import { translate } from "react-i18next";
 import api from "../../util/api";
 import GenericDoughnutChart from "./GenericDoughnutChart";
 
-const ForwardDestinationsChart = ({ t }) => (
+const UpstreamsChart = ({ t }) => (
   <GenericDoughnutChart
     title={t("Queries Answered By Destination")}
-    apiCall={api.getForwardDestinations}
+    apiCall={api.getUpstreams}
+    apiHandler={data =>
+      data.upstreams.map(upstream => ({
+        name: upstream.name,
+        ip: upstream.ip,
+        percent: (upstream.count * 100) / data.total_queries
+      }))
+    }
   />
 );
 
-export default translate("dashboard")(ForwardDestinationsChart);
+export default translate("dashboard")(UpstreamsChart);

src/util/api.js

@@ -1,4 +1,5 @@
 import http from "./http";
+import config from "../config";
 
 export default {
   loggedIn: false,
@@ -22,14 +23,20 @@ export default {
   getQueryTypes() {
     return http.get("stats/query_types");
   },
-  getForwardDestinations() {
-    return http.get("stats/forward_destinations");
+  getUpstreams() {
+    return http.get("stats/upstreams");
   },
   getTopDomains() {
     return http.get("stats/top_domains");
   },
   getTopBlocked() {
-    return http.get("stats/top_blocked");
+    // The API uses a GET parameter to differentiate top domains from top
+    // blocked, but the fake API is not able to handle GET parameters right now.
+    const url = config.fakeAPI
+      ? "stats/top_blocked"
+      : "stats/top_domains?blocked=true";
+
+    return http.get(url);
   },
   getTopClients() {
     return http.get("stats/top_clients");

src/views/Dashboard.js

@@ -13,7 +13,7 @@ import SummaryStats from "../components/dashboard/SummaryStats";
 import QueriesGraph from "../components/dashboard/QueriesGraph";
 import ClientsGraph from "../components/dashboard/ClientsGraph";
 import QueryTypesChart from "../components/dashboard/QueryTypesChart";
-import ForwardDestinationsChart from "../components/dashboard/ForwardDestinationsChart";
+import UpstreamsChart from "../components/dashboard/UpstreamsChart";
 import TopDomains from "../components/dashboard/TopDomains";
 import TopBlocked from "../components/dashboard/TopBlocked";
 import TopClients from "../components/dashboard/TopClients";
@@ -41,7 +41,7 @@ export default () => (
             <QueryTypesChart />
           </div>
           <div className="col-md-6">
-            <ForwardDestinationsChart />
+            <UpstreamsChart />
           </div>
         </div>
         <div className="row">