[Spark] spark中API的使用

1. FileSystem

/** file system API */ import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.conf.Configuration val conf = new Configuration() val path = new Path(dir) val fs: FileSystem = FileSystem.get(path.toUri, conf) if (fs.exists(path)) { colorful_green_println(path.toString + " exists and delete.") fs.delete(path) }

2. CommendLine

import org.apache.commons.cli.{BasicParser, CommandLine, Options} /** Parse the arguments */ val options: Options = new Options() options.addOption("yesterday", true, "The date of yesterday.") options.addOption("window", true, "The window of data we used.") options.addOption("base_dir", true, "The base dir of path") val parser: BasicParser = new BasicParser() val cl: CommandLine = parser.parse(options, args) val yesterday = cl.getOptionValue("yesterday") val window = cl.getOptionValue("window").toInt val base_dir = cl.getOptionValue("base_dir")