auto detect text file encode and convert to utf8

release/v1.15
lunnyxiao 2014-08-20 16:34:48 +08:00
parent a60edbe88c
commit 97fb62f51e
1 changed files with 25 additions and 1 deletions

View File

@ -6,6 +6,7 @@ package repo
import ( import (
"bytes" "bytes"
"fmt"
"io/ioutil" "io/ioutil"
"path" "path"
"path/filepath" "path/filepath"
@ -14,12 +15,30 @@ import (
"github.com/gogits/gogs/modules/base" "github.com/gogits/gogs/modules/base"
"github.com/gogits/gogs/modules/git" "github.com/gogits/gogs/modules/git"
"github.com/gogits/gogs/modules/middleware" "github.com/gogits/gogs/modules/middleware"
"code.google.com/p/mahonia"
"github.com/saintfish/chardet"
) )
const ( const (
HOME base.TplName = "repo/home" HOME base.TplName = "repo/home"
) )
func toUtf8(content []byte) (error, string) {
detector := chardet.NewTextDetector()
result, err := detector.DetectBest(content)
if err != nil {
return err, ""
}
if result.Charset == "utf8" {
return nil, string(content)
}
decoder := mahonia.NewDecoder(result.Charset)
return nil, decoder.ConvertString(string(content))
}
func Home(ctx *middleware.Context) { func Home(ctx *middleware.Context) {
ctx.Data["Title"] = ctx.Repo.Repository.Name ctx.Data["Title"] = ctx.Repo.Repository.Name
@ -98,7 +117,12 @@ func Home(ctx *middleware.Context) {
if readmeExist { if readmeExist {
ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, "")) ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, ""))
} else { } else {
if err, content := toUtf8(buf); err != nil {
fmt.Println("transfer encode error:", err)
ctx.Data["FileContent"] = string(buf) ctx.Data["FileContent"] = string(buf)
} else {
ctx.Data["FileContent"] = content
}
} }
} }
} }