Merge #1442

1442: Soft deprecation for Datasets r=DhairyaLGandhi a=CarloLucibello

Add a soft deprecations path for the datasets I brutally removed in #1377 .
I added the old tests back, verified they passed, then removed them again. 

Fix #1426


Co-authored-by: Carlo Lucibello <carlo.lucibello@gmail.com>
Co-authored-by: Dhairya Gandhi <dhairya@juliacopmuting.com>

Author: 3 people

bors.toml

@@ -1,4 +1,4 @@
 status = [
-  "ci/gitlab%"
+  "buildkite/flux-dot-jl"
 ]
 timeout-sec = 7200

src/data/Data.jl

@@ -7,32 +7,62 @@ include("dataloader.jl")
 export DataLoader
 
 
-## TODO: remove in v0.13 ##############
-module MNIST
-images() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-labels() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-end
-module Iris
-features() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-labels() = error("Dataset is deprecated, use MLDatasets.jl instead.")
+## TODO for v0.13: remove everything below ##############
+## Also remove the following deps:
+## AbstractTrees, ZipFiles, CodecZLib
+
+import ..Flux
+import SHA
+
+deprecation_message() = @warn("Flux's datasets are deprecated, please use the package MLDatasets.jl")
+
+function deps(path...)
+  if isnothing(@__DIR__) # sysimages
+    joinpath("deps", path...)
+  else
+    joinpath(@__DIR__, "..", "..", "deps", path...)
+  end
 end
-module FashionMNIST
-images() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-labels() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-end 
-module CMUDict
-phones() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-symbols() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-rawdict() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-cmudict() = error("Dataset is deprecated, use MLDatasets.jl instead.")
+
+function download_and_verify(url, path, hash)
+    tmppath = tempname()
+    download(url, tmppath)
+    hash_download = open(tmppath) do f
+        bytes2hex(SHA.sha256(f))
+    end
+    if hash_download !== hash
+        msg  = "Hash Mismatch!\n"
+        msg *= "  Expected sha256:   $hash\n"
+        msg *= "  Calculated sha256: $hash_download"
+        error(msg)
+    end
+    mv(tmppath, path; force=true)
 end
-module Sentiment
-train() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-test() = error("Dataset is deprecated, use MLDatasets.jl instead.")
-dev() = error("Dataset is deprecated, use MLDatasets.jl instead.")
+
+function __init__()
+  mkpath(deps())
 end
 
-export MNIST, Iris, FashionMNIST, CMUDict, Sentiment
+include("mnist.jl")
+export MNIST
+
+include("fashion-mnist.jl")
+export FashionMNIST
+
+include("cmudict.jl")
+export CMUDict
+using .CMUDict; export cmudict
+
+include("tree.jl")
+include("sentiment.jl")
+export Sentiment
+
+include("iris.jl")
+export Iris
+
+include("housing.jl")
+export Housing
+
 #########################################
 
-end#module
+end#module

src/data/cmudict.jl

@@ -0,0 +1,77 @@
+module CMUDict
+
+export cmudict
+
+using ..Data: deps, download_and_verify, deprecation_message
+
+const version = "0.7b"
+const cache_prefix = "https://cache.julialang.org"
+
+function load()
+  suffixes_and_hashes = [(""       , "209a8b4cd265013e96f4658632a9878103b0c5abf62b50d4ef3ae1be226b29e4"),
+                        (".phones" , "ffb588a5e55684723582c7256e1d2f9fadb130011392d9e59237c76e34c2cfd6"),
+                        (".symbols", "408ccaae803641c6d7b626b6299949320c2dbca96b2220fd3fb17887b023b027")]
+  if isdir(deps("cmudict"))
+    if all(isfile(deps("cmudict", "cmudict$x")) for (x, _) in suffixes_and_hashes)
+      return
+    end
+  end
+  @info "Downloading CMUDict dataset"
+  mkpath(deps("cmudict"))
+  for (x, hash) in suffixes_and_hashes
+    download_and_verify("$cache_prefix/https://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict/cmudict-$version$x",
+             deps("cmudict", "cmudict$x"), hash)
+  end
+end
+
+"""
+    phones()
+Return a `Vector` containing the phones used in the CMU Pronouncing Dictionary.
+"""
+function phones()
+  deprecation_message()
+  load()
+  Symbol.(first.(split.(split(read(deps("cmudict", "cmudict.phones"),String),
+                        "\n", keepempty = false), "\t")))
+end
+
+"""
+    symbols()
+Return a `Vector` containing the symbols used in the CMU Pronouncing Dictionary.
+A symbol is a phone with optional auxiliary symbols, indicating for example the
+amount of stress on the phone.
+"""
+function symbols()
+  deprecation_message()
+  load()
+  Symbol.(split(read(deps("cmudict", "cmudict.symbols"),String),
+                "\n", keepempty = false))
+end
+
+"""
+    rawdict()
+Return the unfiltered CMU Pronouncing Dictionary.
+"""
+function rawdict()
+  deprecation_message()
+  load()
+  Dict(String(xs[1]) => Symbol.(xs[2:end]) for xs in
+       filter(!isempty, split.(split(read(deps("cmudict", "cmudict"),String), "\n"))))
+end
+
+validword(s) = isascii(s) && occursin(r"^[\w\-\.]+$", s)
+
+"""
+    cmudict()
+Return a filtered CMU Pronouncing Dictionary.
+It is filtered so each word contains only ASCII characters and a combination of
+word characters (as determined by the regex engine using `\\w`), '-' and '.'.
+"""
+function cmudict()
+  deprecation_message()
+  filter(p -> validword(p.first), rawdict())
+end
+
+alphabet() = ['A':'Z'..., '0':'9'..., '_', '-', '.']
+
+end

src/data/fashion-mnist.jl

@@ -0,0 +1,67 @@
+module FashionMNIST
+
+using ..MNIST: gzopen, imageheader, rawimage, labelheader, rawlabel
+using ..Data: download_and_verify, deprecation_message
+
+const dir = if isnothing(@__DIR__)
+    joinpath("deps", "fashion-mnist")
+  else
+    joinpath(@__DIR__, "../../deps/fashion-mnist")
+end
+
+function load()
+  mkpath(dir)
+  cd(dir) do
+    for (file, hash) in [("train-images-idx3-ubyte", "3aede38d61863908ad78613f6a32ed271626dd12800ba2636569512369268a84"),
+                         ("train-labels-idx1-ubyte", "a04f17134ac03560a47e3764e11b92fc97de4d1bfaf8ba1a3aa29af54cc90845"),
+                         ("t10k-images-idx3-ubyte" , "346e55b948d973a97e58d2351dde16a484bd415d4595297633bb08f03db6a073"),
+                         ("t10k-labels-idx1-ubyte" , "67da17c76eaffca5446c3361aaab5c3cd6d1c2608764d35dfb1850b086bf8dd5")]
+      isfile(file) && continue
+      @info "Downloading Fashion-MNIST dataset"
+      download_and_verify("http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/$file.gz", "$file.gz", hash)
+      open(file, "w") do io
+        write(io, gzopen(read, "$file.gz"))
+      end
+    end
+  end
+end
+
+const TRAINIMAGES = joinpath(dir, "train-images-idx3-ubyte")
+const TRAINLABELS = joinpath(dir, "train-labels-idx1-ubyte")
+const TESTIMAGES = joinpath(dir, "t10k-images-idx3-ubyte")
+const TESTLABELS = joinpath(dir, "t10k-labels-idx1-ubyte")
+
+"""
+    images()
+    images(:test)
+Load the Fashion-MNIST images.
+Each image is a 28×28 array of `Gray` colour values
+(see [Colors.jl](https://github.com/JuliaGraphics/Colors.jl)).
+Return the 60,000 training images by default; pass `:test` to retrieve the
+10,000 test images.
+"""
+function images(set = :train)
+  deprecation_message()
+  load()
+  io = IOBuffer(read(set == :train ? TRAINIMAGES : TESTIMAGES))
+  _, N, nrows, ncols = imageheader(io)
+  [rawimage(io) for _ in 1:N]
+end
+
+"""
+    labels()
+    labels(:test)
+Load the labels corresponding to each of the images returned from [`images()`](@ref).
+Each label is a number from 0-9.
+Return the 60,000 training labels by default; pass `:test` to retrieve the
+10,000 test labels.
+"""
+function labels(set = :train)
+  deprecation_message()
+  load()
+  io = IOBuffer(read(set == :train ? TRAINLABELS : TESTLABELS))
+  _, N = labelheader(io)
+  [rawlabel(io) for _ = 1:N]
+end
+
+end

src/data/housing.jl

@@ -0,0 +1,120 @@
+"""
+1. Title: Boston Housing Data
+2. Sources:
+   (a) Origin:  This dataset was taken from the StatLib library which is
+                maintained at Carnegie Mellon University.
+   (b) Creator:  Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the
+                 demand for clean air', J. Environ. Economics & Management,
+                 vol.5, 81-102, 1978.
+   (c) Date: July 7, 1993
+3. Number of Instances: 506
+4. Number of Attributes: 13 continuous attributes (including "class"
+                            attribute "MEDV"), 1 binary-valued attribute.
+5. Attribute Information:
+       1. CRIM      per capita crime rate by town
+       2. ZN        proportion of residential land zoned for lots over
+                    25,000 sq.ft.
+       3. INDUS     proportion of non-retail business acres per town
+       4. CHAS      Charles River dummy variable (= 1 if tract bounds
+                    river; 0 otherwise)
+       5. NOX       nitric oxides concentration (parts per 10 million)
+       6. RM        average number of rooms per dwelling
+       7. AGE       proportion of owner-occupied units built prior to 1940
+       8. DIS       weighted distances to five Boston employment centres
+       9. RAD       index of accessibility to radial highways
+       10. TAX      full-value property-tax rate per 10,000 dollars
+       11. PTRATIO  pupil-teacher ratio by town
+       12. B        1000(Bk - 0.63)^2 where Bk is the proportion of blacks
+                    by town
+       13. LSTAT    % lower status of the population
+       14. MEDV     Median value of owner-occupied homes in 1000's of dollars
+       Downloaded From: https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data
+"""
+module Housing
+
+using DelimitedFiles
+using ..Data: deps, download_and_verify, deprecation_message
+
+#Uncomment if package exists
+#const cache_prefix = "https://cache.julialang.org/"
+const cache_prefix = ""
+
+function load()
+    isfile(deps("housing.data")) && return
+
+    @info "Downloading the Boston housing Dataset"
+    download_and_verify("$(cache_prefix)http://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data",
+                        deps("housing.data"),
+                        "baadf72995725d76efe787b664e1f083388c79ba21ef9a7990d87f774184735a")
+
+    #@info "Download complete. Working on the files"
+    path = deps()
+    isfile(deps("housing.data")) && touch(joinpath(path, "tempfile.data"))
+    open(joinpath(path, "tempfile.data"), "a") do fout
+        open(deps("housing.data"), "r") do fin
+            for line in eachline(fin)
+                line = replace(lstrip(line), r" +" => s",")
+                println(fout, line)
+            end
+        end
+    end
+    mv(joinpath(path, "tempfile.data"), deps("housing.data"), force=true)
+end
+
+"""
+Gets the targets for the Boston housing dataset, a 506 element array listing the targets for each example
+```julia
+julia> using Flux
+julia> target = Flux.Data.Housing.targets()
+julia> summary(target)
+506×1 Array{Float64,2}
+julia> target[1]
+24.0
+"""
+function targets()
+    deprecation_message()
+    load()
+    housing = readdlm(deps("housing.data"), ',')
+    reshape(Vector{Float64}(housing[1:end,end]), (506, 1))
+end
+
+
+"""
+Gets the names of the features provided in the dataset
+"""
+function feature_names()
+    ["crim","zn","indus","chas","nox","rm","age","dis","rad","tax","ptratio","b","lstat"]
+end
+
+
+"""
+Gets the features of the Boston Housing Dataset. This is a 506x13 Matrix of Float64 datatypes.
+The values are in the order ["crim","zn","indus","chas","nox","rm","age","dis","rad","tax","ptratio","b","lstat"].
+It has 506 examples.
+```julia
+julia> using Flux
+julia> features = Flux.Data.Housing.features()
+julia> summary(features)
+506×13 Array{Float64,2}
+julia> features[1, :]
+13-element Array{Float64,1}:
+0.00632
+18.0
+2.31
+0.0
+0.538
+   ⋮
+296.0
+15.3
+396.9
+4.98
+"""
+function features()
+    deprecation_message()
+    load()
+    housing = readdlm(deps("housing.data"), ',')
+    Matrix{Float64}(housing[1:end, 1:13])
+end
+
+
+end