Added streaming support, meta generation for IO functions

.travis.yml

@@ -15,3 +15,6 @@ notifications:
 #script:
 #  - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
 #  - julia -e 'Pkg.clone(pwd()); Pkg.build("LasIO"); Pkg.test("LasIO"; coverage=true)'
+
+after_success:
+- julia -e 'cd(Pkg.dir("LasIO")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())'

README.md

@@ -16,6 +16,15 @@ header, points = load("test.las")
 
 where `header` is of type `LasHeader`, and, if it is point format 3, `points` is a `Vector{LasPoint3}`. `LasPoint3` is an immutable that directly corresponds to the binary data in the LAS file. Use functions like `xcoord(p::LasPoint, header::LasHeader)` to take out the desired items in the point.
 
+If the file does not fit into memory, it can be memory mapped using
+
+```julia
+using FileIO, LasIO
+header, points = load("test.las", mmap=true)
+```
+
+where `points` is now a memory mapped `PointVector{LasPoint3}` which behaves in the same way as the `Vector{LasPoint3}`, but reads the points on the fly from disk when indexed, not allocating the complete vector beforehand.
+
 See `test/runtests.jl` for other usages.
 
 ## LAZ support

REQUIRE

@@ -3,3 +3,4 @@ ColorTypes 0.3.3
 FixedPointNumbers 0.3.2
 FileIO 0.3.0
 GeometryTypes 0.4.0
+Compat 0.29

src/LasIO.jl

@@ -2,12 +2,15 @@ __precompile__()
 
 module LasIO
 
+using Base.Meta
+using Compat
 using FileIO
 using FixedPointNumbers
 using ColorTypes
 using GeometryTypes # for conversion
 
 export
+
     # Types
     LasHeader,
     LasVariableLengthRecord,
@@ -16,9 +19,11 @@ export
     LasPoint1,
     LasPoint2,
     LasPoint3,
+    PointVector,
 
     # Functions on LasHeader
     update!,
+    pointformat,
 
     # Functions on LasPoint
     return_number,
@@ -48,6 +53,7 @@ export
 
 include("vlrs.jl")
 include("header.jl")
+include("meta.jl")
 include("point.jl")
 include("util.jl")
 include("fileio.jl")

src/fileio.jl

@@ -17,13 +17,14 @@ end
 # skip the LAS file's magic four bytes, "LASF"
 skiplasf(s::Union{Stream{format"LAS"}, Stream{format"LAZ"}, IO}) = read(s, UInt32)
 
-function load(f::File{format"LAS"})
+function load(f::File{format"LAS"}; mmap=false)
     open(f) do s
-        load(s)
+        load(s; mmap=mmap)
     end
 end
 
-function load(s::Union{Stream{format"LAS"}, Pipe})
+# Load pipe separately since it can't be memory mapped
+function load(s::Pipe)
     skiplasf(s)
     header = read(s, LasHeader)
 
@@ -36,6 +37,27 @@ function load(s::Union{Stream{format"LAS"}, Pipe})
     header, pointdata
 end
 
+function load(s::Stream{format"LAS"}; mmap=false)
+    skiplasf(s)
+    header = read(s, LasHeader)
+
+    n = header.records_count
+    pointtype = pointformat(header)
+
+    if mmap
+        pointsize = Int(header.data_record_length)
+        pointbytes = Mmap.mmap(s.io, Vector{UInt8}, n*pointsize, position(s))
+        pointdata = PointVector{pointtype}(pointbytes, pointsize)
+    else
+        pointdata = Vector{pointtype}(n)
+        for i=1:n
+            pointdata[i] = read(s, pointtype)
+        end
+    end
+
+    header, pointdata
+end
+
 function load(f::File{format"LAZ"})
     # read las from laszip, which decompresses to stdout
     open(`laszip -olas -stdout -i $(filename(f))`) do s
@@ -55,13 +77,13 @@ function read_header(s::IO)
     read(s, LasHeader)
 end
 
-function save(f::File{format"LAS"}, header::LasHeader, pointdata::Vector{T}) where T <: LasPoint
+function save(f::File{format"LAS"}, header::LasHeader, pointdata::AbstractVector{<:LasPoint})
     open(f, "w") do s
         save(s, header, pointdata)
     end
 end
 
-function save(s::Stream{format"LAS"}, header::LasHeader, pointdata::Vector{T}) where T <: LasPoint
+function save(s::Stream{format"LAS"}, header::LasHeader, pointdata::AbstractVector{<:LasPoint})
     # checks
     header_n = header.records_count
     n = length(pointdata)
@@ -78,7 +100,7 @@ function save(s::Stream{format"LAS"}, header::LasHeader, pointdata::Vector{T}) w
     end
 end
 
-function save(f::File{format"LAZ"}, header::LasHeader, pointdata::Vector{T}) where T <: LasPoint
+function save(f::File{format"LAZ"}, header::LasHeader, pointdata::AbstractVector{<:LasPoint})
     # pipes las to laszip to write laz
     open(`laszip -olaz -stdin -o $(filename(f))`, "w") do s
         savebuf(s, header, pointdata)
@@ -88,7 +110,7 @@ end
 # Uses a buffered write to the stream.
 # For saving to LAS this does not increase speed,
 # but it speeds up a lot when the result is piped to laszip.
-function savebuf(s::IO, header::LasHeader, pointdata::Vector{T}) where T <: LasPoint
+function savebuf(s::IO, header::LasHeader, pointdata::AbstractVector{<:LasPoint})
     # checks
     header_n = header.records_count
     n = length(pointdata)

src/meta.jl

@@ -0,0 +1,63 @@
+"Generate read (unpack) method for structs."
+function generate_read(T::Type)
+    fc = @compat fieldcount(T)
+    types = [fieldtype(T, i) for i = 1:fc]
+
+    # Create unpack function expression
+    function_expression = :(function Base.read(io::IO, t::Type{$T}) end)
+
+    # Create Type call expression and add parameters
+    type_expression = :(($T)())
+    for t in types
+        read_expression = :(read(io, $t))
+        append!(type_expression.args, 0)  # dummy with known length
+        type_expression.args[end] = read_expression
+    end
+
+    # Replace empty function body with Type call
+    function_expression.args[2] = type_expression
+
+    eval(function_expression)
+end
+
+"Generate write (pack) method for structs."
+function generate_write(T::Type)
+    # Create pack function expression
+    function_expression = :(function Base.write(io::IO, T::$T) end)
+
+    body_expression = quote end
+    for t in fieldnames(T)
+        append!(body_expression.args, 0)  # dummy with known length
+        write_expression = :(write(io, T.$t))
+        body_expression.args[end] = write_expression
+    end
+
+    # Return nothing at the end
+    append!(body_expression.args, 0)  # dummy with known length
+    body_expression.args[end] = :(nothing)
+
+    # Replace empty function body with write calls
+    function_expression.args[2] = body_expression
+
+    eval(function_expression)
+end
+
+function generate_io(T::Type)
+    generate_read(T)
+    generate_write(T)
+end
+
+"""Generate IO expressions macro."""
+macro gen_io(typ::Expr)
+    T = typ.args[2]
+    if isexpr(T, :(<:))
+        T = T.args[1]
+    end
+    if isexpr(T, :curly)
+        T = T.args[1]
+    end
+
+    ret = Expr(:toplevel, :(Base.@__doc__ $(typ)))
+    push!(ret.args, :(generate_io($T)))
+    return esc(ret)
+end