Skip to content

Change URL #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Change URL #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c972d4c
Change URL source
Guillaume001 Jul 27, 2017
e68ba5d
URL change
Guillaume001 Jul 28, 2017
2c63511
DLL
Guillaume001 Jul 31, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added BeatlesBlog.SimConnect.dll
View file
Open in desktop
Binary file not shown.
Binary file added Catfood.Shapefile.dll
View file
Open in desktop
Binary file not shown.
121 changes: 76 additions & 45 deletions FSActiveFires/MODISHotspots.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,83 +1,96 @@
using Catfood.Shapefile;
using Catfood.Shapefile;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Net;

namespace FSActiveFires {
class MODISHotspots {
namespace FSActiveFires
{
class MODISHotspots
{
public Dictionary<string, string> datasets { get; private set; }
public HashSet<Hotspot> hotspots { get; private set; }
private string tempDirectory;
private Log log;

public MODISHotspots() {
public MODISHotspots()
{
hotspots = new HashSet<Hotspot>();
datasets = new Dictionary<string, string>();
log = Log.Instance;

datasets.Add("World", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/Global_24h.{1}");
datasets.Add("Alaska", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/Alaska_24h.{1}");
datasets.Add("Australia and New Zealand", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/Australia_and_New_Zealand_24h.{1}");
datasets.Add("Canada", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/Canada_24h.{1}");
datasets.Add("Central America", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/Central_America_24h.{1}");
datasets.Add("Europe", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/Europe_24h.{1}");
datasets.Add("Northern and Central Africa", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/Northern_and_Central_Africa_24h.{1}");
datasets.Add("Russia and Asia", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/Russia_and_Asia_24h.{1}");
datasets.Add("South America", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/South_America_24h.{1}");
datasets.Add("South Asia", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/South_Asia_24h.{1}");
datasets.Add("South East Asia", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/SouthEast_Asia_24h.{1}");
datasets.Add("Southern Africa", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/Southern_Africa_24h.{1}");
datasets.Add("U.S.A. (Conterminous) and Hawaii", "https://firms.modaps.eosdis.nasa.gov/active_fire/{0}/USA_contiguous_and_Hawaii_24h.{1}");
datasets.Add("World", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_Global_24h.{1}");
datasets.Add("Alaska", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_Alaska_24h.{1}");
datasets.Add("Australia and New Zealand", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_Australia_and_New_Zealand_24h.{1}");
datasets.Add("Canada", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_Canada_24h.{1}");
datasets.Add("Central America", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_Central_America_24h.{1}");
datasets.Add("Europe", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_Europe_24h.{1}");
datasets.Add("Northern and Central Africa", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_Northern_and_Central_Africa_24h.{1}");
datasets.Add("Russia and Asia", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_Russia_and_Asia_24h.{1}");
datasets.Add("South America", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_South_America_24h.{1}");
datasets.Add("South Asia", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_South_Asia_24h.{1}");
datasets.Add("South East Asia", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_SouthEast_Asia_24h.{1}");
datasets.Add("Southern Africa", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_Southern_Africa_24h.{1}");
datasets.Add("U.S.A. (Conterminous) and Hawaii", "https://firms.modaps.eosdis.nasa.gov/active_fire/c6/{0}/MODIS_C6_USA_contiguous_and_Hawaii_24h.{1}");

tempDirectory = CreateTemporaryDirectory();
}

private string CreateTemporaryDirectory() {
private string CreateTemporaryDirectory()
{
string tempDirectory = Path.Combine(Path.GetTempPath(), "FSActiveFires", Path.GetRandomFileName());
log.Info(string.Format("Create temporary directory: {0}", tempDirectory));
Directory.CreateDirectory(tempDirectory);
return tempDirectory;
}

public void RemoveTemporaryDirectory() {
public void RemoveTemporaryDirectory()
{
string tempDirectoryRoot = Path.Combine(Path.GetTempPath(), "FSActiveFires");
if (Directory.Exists(tempDirectoryRoot)) {
if (Directory.Exists(tempDirectoryRoot))
{
log.Info(string.Format("Delete temporary directory: {0}", tempDirectoryRoot));
Directory.Delete(tempDirectoryRoot, true);
}
}

public void LoadData(string datasetFormatString) {
try {
public void LoadData(string datasetFormatString)
{
try
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
LoadShapefileHotspots(DownloadShapefileData(datasetFormatString));
}
catch (System.Data.OleDb.OleDbException ex) {
catch (System.Data.OleDb.OleDbException ex)
{
log.Warning(string.Format("Unable to parse shapefile; attempting to use CSV.\r\n{0}", ex.ToString()));
LoadCsvHotspots(DownloadCsvData(datasetFormatString));
}
}

private string DownloadShapefileData(string datasetFormatString) {
private string DownloadShapefileData(string datasetFormatString)
{
string webUrl = string.Format(datasetFormatString, "shapes/zips", "zip");
string zipFileName = webUrl.Substring(webUrl.LastIndexOf('/') + 1, webUrl.Length - webUrl.LastIndexOf('/') - 1);
string zipFilePath = Path.Combine(tempDirectory, zipFileName);
string shapefilePath = zipFilePath.Substring(0, zipFilePath.Length - 3) + "shp";

if (File.Exists(shapefilePath)) {
if (File.Exists(shapefilePath))
{
log.Info(string.Format("SHP already exists: {0}", shapefilePath));
return shapefilePath;
}

using (WebClient webClient = new WebClient()) {
using (WebClient webClient = new WebClient())
{
log.Info(string.Format("Download ZIP: {0} -> {1}", webUrl, zipFilePath));
webClient.DownloadFile(webUrl, zipFilePath);
}

if (!File.Exists(zipFilePath)) {
if (!File.Exists(zipFilePath))
{
throw new FileNotFoundException("ZIP file was not downloaded.");
}

Expand All @@ -86,44 +99,56 @@ private string DownloadShapefileData(string datasetFormatString) {
log.Info(string.Format("Delete ZIP: {0}", zipFilePath));
File.Delete(zipFilePath);

if (File.Exists(shapefilePath)) {
if (File.Exists(shapefilePath))
{
return shapefilePath;
}
else {
else
{
throw new FileNotFoundException("Shapefile was not downloaded or extracted.");
}
}

private string DownloadCsvData(string datasetFormatString) {
private string DownloadCsvData(string datasetFormatString)
{
string webUrl = string.Format(datasetFormatString, "text", "csv");
string fileName = webUrl.Substring(webUrl.LastIndexOf('/') + 1, webUrl.Length - webUrl.LastIndexOf('/') - 1);
string filePath = Path.Combine(tempDirectory, fileName);

if (File.Exists(filePath)) {
if (File.Exists(filePath))
{
log.Info(string.Format("CSV already exists: {0}", filePath));
return filePath;
}

using (WebClient webClient = new WebClient()) {
using (WebClient webClient = new WebClient())
{
log.Info(string.Format("Download CSV: {0} -> {1}", webUrl, filePath));
webClient.DownloadFile(webUrl, filePath);
}

if (File.Exists(filePath)) {
if (File.Exists(filePath))
{
return filePath;
}
else {
else
{
throw new FileNotFoundException("CSV file was not downloaded.");
}
}

private void LoadShapefileHotspots(string shapefilePath) {
private void LoadShapefileHotspots(string shapefilePath)
{
log.Info(string.Format("Parsing shapefile: {0}", shapefilePath));
using (Shapefile shp = new Shapefile(shapefilePath)) {
foreach (Shape shape in shp) {
if (shape.Type == ShapeType.Point) {
using (Shapefile shp = new Shapefile(shapefilePath))
{
foreach (Shape shape in shp)
{
if (shape.Type == ShapeType.Point)
{
int confidence;
if (int.TryParse(shape.GetMetadata("confidence"), out confidence)) {
if (int.TryParse(shape.GetMetadata("confidence"), out confidence))
{
ShapePoint shapePoint = shape as ShapePoint;
hotspots.Add(new Hotspot(shapePoint.Point.Y, shapePoint.Point.X, confidence));
}
Expand All @@ -133,25 +158,31 @@ private void LoadShapefileHotspots(string shapefilePath) {
log.Info(string.Format("Cumulative hotspots parsed: {0}", hotspots.Count));
}

private void LoadCsvHotspots(string csvPath) {
private void LoadCsvHotspots(string csvPath)
{
log.Info(string.Format("Parsing CSV: {0}", csvPath));
using (StreamReader sr = new StreamReader(csvPath)) {
using (StreamReader sr = new StreamReader(csvPath))
{
string line = sr.ReadLine();
var fields = line.Split(',');
if (fields[0].Equals("latitude") && fields[1].Equals("longitude") && fields[8].Equals("confidence")) {
while ((line = sr.ReadLine()) != null) {
if (fields[0].Equals("latitude") && fields[1].Equals("longitude") && fields[8].Equals("confidence"))
{
while ((line = sr.ReadLine()) != null)
{
fields = line.Split(',');
double lat;
double lon;
int confidence;
if (double.TryParse(fields[0], NumberStyles.Number, CultureInfo.InvariantCulture, out lat) &&
double.TryParse(fields[1], NumberStyles.Number, CultureInfo.InvariantCulture, out lon) &&
int.TryParse(fields[8], NumberStyles.Number, CultureInfo.InvariantCulture, out confidence)) {
int.TryParse(fields[8], NumberStyles.Number, CultureInfo.InvariantCulture, out confidence))
{
hotspots.Add(new Hotspot(lat, lon, confidence));
}
}
}
else {
else
{
throw new FileFormatException("Unexpected data format.");
}
}
Expand Down